Delete full_eval_results.json

full_eval_results.json

@@ -1,2289 +0,0 @@
-{
-  "teacher": "Qwen/Qwen3.5-4B",
-  "max_new_tokens": 512,
-  "block_seed": 3735928559,
-  "tensor_parallel_size": 2,
-  "n_prompts": 10,
-  "n_prompts_filtered": 0,
-  "min_completion_tokens": 10,
-  "students": {
-    "vuhaian/266k_no_format": {
-      "chat_probe": {
-        "pass": true,
-        "reason": "",
-        "prompts_tested": 4,
-        "prompts_terminated": 4,
-        "prompts_non_empty": 4,
-        "mean_gen_tokens": 54.5,
-        "mean_content_chars": 194.0,
-        "mean_reasoning_fraction": 0.0,
-        "samples": [
-          {
-            "prompt": "hi",
-            "gen_tokens": 6,
-            "terminated": true,
-            "content_chars": 10,
-            "reasoning_frac": 0.0,
-            "content_preview": "Roarbox #5"
-          },
-          {
-            "prompt": "What is 2+2?",
-            "gen_tokens": 3,
-            "terminated": true,
-            "content_chars": 2,
-            "reasoning_frac": 0.0,
-            "content_preview": "22"
-          },
-          {
-            "prompt": "Say hello in one word.",
-            "gen_tokens": 206,
-            "terminated": true,
-            "content_chars": 762,
-            "reasoning_frac": 0.0,
-            "content_preview": "Here is a unique, classic combinatorics, often referred to as \"handshake problem"
-          },
-          {
-            "prompt": "Reply with just the word: yes",
-            "gen_tokens": 3,
-            "terminated": true,
-            "content_chars": 2,
-            "reasoning_frac": 0.0,
-            "content_preview": "13"
-          }
-        ]
-      },
-      "capability": {
-        "n": 36,
-        "correct": 8,
-        "pass_frac": 0.222,
-        "teacher_pass_frac": 0.889,
-        "items": [
-          {
-            "q": "What is the name of the closest star to Earth? One word.",
-            "expected": "sun",
-            "kind": "word",
-            "pred": "the",
-            "ok": false,
-            "tail": "**carousel** of Earth, meaning all levels are in the lunar eclipse. The most common name for the planet is **sunkenes**."
-          },
-          {
-            "q": "What color do you get by mixing blue and yellow? One word.",
-            "expected": "green",
-            "kind": "word",
-            "pred": "the",
-            "ok": false,
-            "tail": "The primary difference between blue and orange and red? Answer in orange."
-          },
-          {
-            "q": "How many letters are in the word 'banana'? Answer with only the number.",
-            "expected": "6",
-            "kind": "int",
-            "pred": "2",
-            "ok": false,
-            "tail": "2"
-          },
-          {
-            "q": "What is the freezing point of water in Fahrenheit? Answer with only the number.",
-            "expected": "32",
-            "kind": "int",
-            "pred": "13",
-            "ok": false,
-            "tail": "13"
-          },
-          {
-            "q": "Which month has exactly 28 or 29 days? One word.",
-            "expected": "february",
-            "kind": "word",
-            "pred": "",
-            "ok": false,
-            "tail": "29"
-          },
-          {
-            "q": "Is 29 a prime number? Answer yes or no.",
-            "expected": "yes",
-            "kind": "yesno",
-            "pred": "",
-            "ok": false,
-            "tail": "11"
-          },
-          {
-            "q": "What is the chemical symbol for water? One word.",
-            "expected": "h2o",
-            "kind": "word",
-            "pred": "the",
-            "ok": false,
-            "tail": "ts molecules.\n\nIn liquid water, we can use the chemical formula for the population in Mexico to \"danger zone\" between 40"
-          },
-          {
-            "q": "What does 'len([1, 2, 3, 4])' evaluate to in Python? Answer with only the number.",
-            "expected": "4",
-            "kind": "int",
-            "pred": "2",
-            "ok": false,
-            "tail": "2"
-          },
-          {
-            "q": "Answer with exactly three words (no more, no less) describing a cat.",
-            "expected": "word_count",
-            "kind": "word_count",
-            "pred": "1.  **Analyze the Request:**\n    *   Task: Explain why a cat is using a combination of three vegetables.\n    *   Constraint: Exactly two sentences.\n\n2.  **Define \"Dream Forest\":**",
-            "ok": false,
-            "tail": "t is using a combination of three vegetables.\n    *   Constraint: Exactly two sentences.\n\n2.  **Define \"Dream Forest\":**"
-          },
-          {
-            "q": "What is the past tense of 'run'? One word.",
-            "expected": "ran",
-            "kind": "word",
-            "pred": "the",
-            "ok": false,
-            "tail": "The task is complete. fin."
-          },
-          {
-            "q": "What is the opposite of 'cold'? One word.",
-            "expected": "hot",
-            "kind": "word_alt",
-            "pred": "the",
-            "ok": false,
-            "tail": "**verify that individual components of your code, such as your software engineering, data processing, and other critical"
-          },
-          {
-            "q": "Which continent is Egypt on? One word.",
-            "expected": "africa",
-            "kind": "word",
-            "pred": "france",
-            "ok": false,
-            "tail": "France, Germany, Italy, Spain, United Kingdom"
-          },
-          {
-            "q": "How many vowels are in the word 'puzzle'? Answer with only the number.",
-            "expected": "2",
-            "kind": "int",
-            "pred": "2",
-            "ok": true,
-            "tail": "2"
-          },
-          {
-            "q": "How many even numbers are in this list: 73, 50, 51, 55, 94, 53, 74? Answer with only the number.",
-            "expected": "3",
-            "kind": "int",
-            "pred": "1",
-            "ok": false,
-            "tail": "1"
-          },
-          {
-            "q": "How many even numbers are in this list: 52, 61, 84, 81, 47, 35, 69? Answer with only the number.",
-            "expected": "2",
-            "kind": "int",
-            "pred": "4",
-            "ok": false,
-            "tail": "4"
-          },
-          {
-            "q": "Which is larger: 2921 or 5901? Answer with just the number.",
-            "expected": "5901",
-            "kind": "int",
-            "pred": "9211",
-            "ok": false,
-            "tail": "9211"
-          },
-          {
-            "q": "What is 484 + 398? Answer with only the number.",
-            "expected": "882",
-            "kind": "int",
-            "pred": "444",
-            "ok": false,
-            "tail": "444"
-          },
-          {
-            "q": "What is 56 + 231? Answer with only the number.",
-            "expected": "287",
-            "kind": "int",
-            "pred": "631",
-            "ok": false,
-            "tail": "631"
-          },
-          {
-            "q": "How many even numbers are in this list: 63, 81, 71, 32, 1, 64, 55, 46? Answer with only the number.",
-            "expected": "3",
-            "kind": "int",
-            "pred": "3",
-            "ok": true,
-            "tail": "3"
-          },
-          {
-            "q": "What is the minimum of: 28, 22, 35, 23, 40? Answer with only the number.",
-            "expected": "22",
-            "kind": "int",
-            "pred": "23",
-            "ok": false,
-            "tail": "23"
-          },
-          {
-            "q": "What is 447 + 307? Answer with only the number.",
-            "expected": "754",
-            "kind": "int",
-            "pred": "474",
-            "ok": false,
-            "tail": "474"
-          },
-          {
-            "q": "How many characters are in the word 'circle'? Answer with only the number.",
-            "expected": "6",
-            "kind": "int",
-            "pred": "6",
-            "ok": true,
-            "tail": "6"
-          },
-          {
-            "q": "Is 66395 even or odd? Answer with one word: 'even' or 'odd'.",
-            "expected": "odd",
-            "kind": "word",
-            "pred": "even",
-            "ok": false,
-            "tail": "even"
-          },
-          {
-            "q": "How many even numbers are in this list: 48, 23, 14, 1, 55, 6? Answer with only the number.",
-            "expected": "3",
-            "kind": "int",
-            "pred": "3",
-            "ok": true,
-            "tail": "3"
-          },
-          {
-            "q": "What is 132 / 6? Answer with only the number.",
-            "expected": "22",
-            "kind": "int",
-            "pred": "13",
-            "ok": false,
-            "tail": "13"
-          },
-          {
-            "q": "What is 5 to the power of 3? Answer with only the number.",
-            "expected": "125",
-            "kind": "int",
-            "pred": "25",
-            "ok": false,
-            "tail": "25"
-          },
-          {
-            "q": "Is 67620 even or odd? Answer with one word: 'even' or 'odd'.",
-            "expected": "even",
-            "kind": "word",
-            "pred": "even",
-            "ok": true,
-            "tail": "even"
-          },
-          {
-            "q": "Which is larger: 9038 or 4628? Answer with just the number.",
-            "expected": "9038",
-            "kind": "int",
-            "pred": "9038",
-            "ok": true,
-            "tail": "9038"
-          },
-          {
-            "q": "Is 39 divisible by 7? Answer yes or no.",
-            "expected": "no",
-            "kind": "yesno",
-            "pred": "no",
-            "ok": true,
-            "tail": "no"
-          },
-          {
-            "q": "What is 4 to the power of 2? Answer with only the number.",
-            "expected": "16",
-            "kind": "int",
-            "pred": "23",
-            "ok": false,
-            "tail": "23"
-          },
-          {
-            "q": "What is the minimum of: 5, 87, 48, 68, 37, 32? Answer with only the number.",
-            "expected": "5",
-            "kind": "int",
-            "pred": "3",
-            "ok": false,
-            "tail": "3"
-          },
-          {
-            "q": "What is 9 * 15? Answer with only the number.",
-            "expected": "135",
-            "kind": "int",
-            "pred": "15",
-            "ok": false,
-            "tail": "15"
-          },
-          {
-            "q": "What is 6 * 6? Answer with only the number.",
-            "expected": "36",
-            "kind": "int",
-            "pred": "60",
-            "ok": false,
-            "tail": "60"
-          },
-          {
-            "q": "Is 31857 even or odd? Answer with one word: 'even' or 'odd'.",
-            "expected": "odd",
-            "kind": "word",
-            "pred": "even",
-            "ok": false,
-            "tail": "even"
-          },
-          {
-            "q": "What is 40 / 8? Answer with only the number.",
-            "expected": "5",
-            "kind": "int",
-            "pred": "4",
-            "ok": false,
-            "tail": "4"
-          },
-          {
-            "q": "Is 82360 even or odd? Answer with one word: 'even' or 'odd'.",
-            "expected": "even",
-            "kind": "word",
-            "pred": "even",
-            "ok": true,
-            "tail": "even"
-          }
-        ]
-      },
-      "judge_probe_meta": {
-        "n_prompts": 16,
-        "mean_gen_tokens": 87.4,
-        "collected_at": 127.3
-      },
-      "chat_turns_probe_meta": {
-        "n_convos": 6,
-        "n_turns": 3,
-        "mean_gen_tokens": 186.1,
-        "collected_at": 320.1
-      },
-      "activation_fingerprint": {
-        "layer_fingerprints": {
-          "0": [
-            -0.02358,
-            0.023047,
-            -0.018652,
-            -0.06203,
-            0.065354,
-            0.105742,
-            0.131179,
-            0.005227,
-            0.120648,
-            -0.158617,
-            -0.058113,
-            0.084517,
-            -0.026138,
-            -0.063506,
-            -0.077578,
-            -0.034811,
-            -0.074237,
-            -0.006084,
-            -0.05654,
-            -0.030059,
-            -0.103639,
-            -0.117317,
-            0.124655,
-            -0.088633,
-            0.096639,
-            0.116544,
-            -0.076829,
-            0.117099,
-            0.032611,
-            -0.216403,
-            -0.114123,
-            -0.039659,
-            0.141624,
-            -0.079809,
-            0.003829,
-            0.14665,
-            0.150806,
-            -0.108851,
-            -0.042686,
-            -0.060563,
-            0.179756,
-            0.107547,
-            -0.001415,
-            -0.101834,
-            0.048482,
-            -0.06239,
-            0.094574,
-            -0.187087,
-            -0.05525,
-            0.071986,
-            0.030619,
-            -0.038882,
-            -0.111968,
-            -0.207109,
-            0.024401,
-            -0.118496,
-            0.092319,
-            0.051842,
-            0.100322,
-            -0.167014,
-            0.087614,
-            -0.000598,
-            0.051001,
-            0.045138,
-            0.013818,
-            -0.144281,
-            -0.073048,
-            -0.046344,
-            0.016473,
-            0.068888,
-            0.011685,
-            -0.004957,
-            -0.097593,
-            0.08579,
-            0.073644,
-            0.191274,
-            0.158977,
-            -0.074912,
-            0.005852,
-            0.156189,
-            -0.101118,
-            0.003868,
-            -0.031783,
-            -0.213898,
-            -0.023444,
-            -0.117981,
-            -0.171241,
-            0.123527,
-            0.023992,
-            -0.109726,
-            0.096026,
-            -0.021349,
-            -0.153388,
-            -0.091153,
-            0.069493,
-            0.02735,
-            0.032176,
-            0.026836,
-            -0.198828,
-            0.019355,
-            0.021652,
-            0.050302,
-            -0.029157,
-            -0.012547,
-            -0.017701,
-            0.00725,
-            0.01213,
-            -0.083942,
-            -0.0002,
-            0.170217,
-            0.006994,
-            -0.067303,
-            0.046642,
-            0.010163,
-            -0.041537,
-            0.048544,
-            0.081023,
-            0.020891,
-            0.054447,
-            0.089388,
-            -0.263624,
-            -0.114355,
-            -0.021575,
-            0.192974,
-            -0.089267,
-            0.049052,
-            -0.032412,
-            -0.111175
-          ],
-          "8": [
-            -0.356313,
-            0.542448,
-            -0.026062,
-            -0.247746,
-            0.047553,
-            0.173991,
-            0.250737,
-            0.043759,
-            0.869366,
-            -0.387574,
-            0.275757,
-            0.337193,
-            -0.166669,
-            -0.013,
-            -0.360152,
-            -0.22412,
-            -0.063324,
-            -0.424334,
-            -0.182127,
-            -0.071875,
-            -0.269472,
-            0.023085,
-            0.218655,
-            -0.508551,
-            0.317973,
-            0.412306,
-            0.077114,
-            0.314127,
-            0.944412,
-            -0.25864,
-            -0.213614,
-            -0.207512,
-            -0.071897,
-            -0.547037,
-            -0.045923,
-            -0.048846,
-            0.67462,
-            -0.14531,
-            0.09615,
-            0.158321,
-            0.431241,
-            0.245608,
-            0.08166,
-            -0.157362,
-            -0.560843,
-            -0.188654,
-            0.022864,
-            -0.571392,
-            0.312103,
-            0.116028,
-            -0.16853,
-            -0.077114,
-            0.015948,
-            -0.850711,
-            -0.053794,
-            -0.56074,
-            0.35169,
-            0.127887,
-            0.140828,
-            -0.331677,
-            0.655454,
-            -0.252755,
-            0.190753,
-            0.268501,
-            0.160194,
-            -0.264688,
-            -0.30078,
-            -0.127256,
-            0.611192,
-            0.049363,
-            -0.307033,
-            -0.000383,
-            0.053589,
-            -0.251357,
-            0.460809,
-            0.401471,
-            0.342244,
-            -0.28945,
-            -0.332274,
-            0.470964,
-            -0.339503,
-            -0.334885,
-            0.184045,
-            -0.635249,
-            -0.127951,
-            -0.268283,
-            -0.202101,
-            0.175005,
-            -0.069824,
-            -0.513116,
-            -0.112091,
-            -0.270226,
-            -0.496494,
-            -0.23215,
-            -0.168239,
-            0.132642,
-            -0.103555,
-            0.013657,
-            -0.044037,
-            -0.103405,
-            0.04829,
-            0.050528,
-            -0.409297,
-            -0.080933,
-            -0.062962,
-            -0.570068,
-            0.111535,
-            0.604951,
-            -0.358704,
-            0.226629,
-            0.023521,
-            0.10636,
-            0.308478,
-            0.149058,
-            -0.132217,
-            -0.072786,
-            0.368518,
-            -0.137814,
-            0.223361,
-            -0.176459,
-            -0.841371,
-            0.076617,
-            -0.363967,
-            0.333049,
-            -0.697313,
-            0.267288,
-            -0.497775,
-            -0.773132
-          ],
-          "16": [
-            -0.553848,
-            0.663364,
-            0.247362,
-            -0.031048,
-            0.029107,
-            -0.069239,
-            0.319555,
-            -0.204498,
-            0.696275,
-            -0.55421,
-            0.147498,
-            0.171642,
-            -0.402901,
-            -0.658237,
-            -0.056252,
-            -1.144753,
-            1.411752,
-            -0.208825,
-            0.150669,
-            0.098111,
-            0.262871,
-            0.740419,
-            0.555316,
-            -0.53737,
-            0.757213,
-            0.237095,
-            -0.1508,
-            0.682908,
-            -2.698931,
-            -0.537254,
-            -0.370926,
-            -0.072592,
-            -0.187218,
-            0.316548,
-            0.259634,
-            -0.045607,
-            0.489293,
-            -0.25361,
-            -0.156073,
-            0.021973,
-            0.020636,
-            0.516373,
-            -0.730363,
-            0.158852,
-            -0.244986,
-            -0.852696,
-            -0.341619,
-            -0.524388,
-            0.493624,
-            0.19849,
-            -0.069968,
-            -0.447185,
-            -0.384867,
-            -0.200382,
-            -0.327482,
-            -0.420265,
-            0.238882,
-            0.211189,
-            0.245478,
-            -0.46331,
-            0.610028,
-            -0.157139,
-            0.04278,
-            -0.028567,
-            0.059746,
-            -0.875951,
-            0.288316,
-            -2.8e-05,
-            0.246938,
-            -0.671251,
-            0.089834,
-            0.046384,
-            0.200573,
-            -0.568695,
-            -0.035238,
-            0.44219,
-            0.340631,
-            0.345957,
-            -0.575619,
-            0.332026,
-            -0.02053,
-            -0.681751,
-            0.585333,
-            -0.359494,
-            0.2522,
-            -0.196143,
-            0.346302,
-            0.047533,
-            -0.387082,
-            0.414155,
-            0.080459,
-            -0.197517,
-            0.037546,
-            0.206393,
-            -0.200278,
-            0.622899,
-            -0.400144,
-            0.019957,
-            -0.016437,
-            -0.063687,
-            0.22555,
-            -0.29304,
-            -0.390129,
-            0.104182,
-            -0.004765,
-            -0.249057,
-            -0.042419,
-            0.310793,
-            -0.222476,
-            0.093852,
-            0.041026,
-            -0.53854,
-            0.090736,
-            0.257589,
-            0.431975,
-            -0.162952,
-            0.106623,
-            -0.306464,
-            0.294026,
-            -0.285186,
-            0.303658,
-            0.617457,
-            -0.199019,
-            0.578837,
-            -0.193669,
-            -0.127395,
-            -0.586367,
-            -0.779146
-          ],
-          "23": [
-            -0.258527,
-            0.896605,
-            0.77685,
-            0.033634,
-            0.711418,
-            0.002111,
-            -0.400077,
-            -0.4999,
-            0.499398,
-            -0.568787,
-            0.429323,
-            -0.28107,
-            -0.041556,
-            -0.483025,
-            -0.571643,
-            -0.72305,
-            4.596298,
-            -0.205575,
-            0.185375,
-            0.362442,
-            0.329307,
-            0.077613,
-            0.757329,
-            0.431228,
-            0.732175,
-            -0.380133,
-            -0.174373,
-            1.007456,
-            -7.00988,
-            0.242204,
-            -0.680006,
-            -0.472774,
-            0.054452,
-            0.244063,
-            0.278304,
-            0.031637,
-            -0.005795,
-            0.403786,
-            0.224216,
-            -0.417353,
-            -0.284475,
-            0.733055,
-            -0.56216,
-            0.477628,
-            -0.349799,
-            -2.79653,
-            -1.267487,
-            -0.661888,
-            0.497994,
-            0.297001,
-            -0.50287,
-            -0.271973,
-            -0.160612,
-            0.996306,
-            0.116682,
-            -0.478855,
-            0.437648,
-            0.414956,
-            0.439904,
-            -0.160444,
-            0.970315,
-            -0.280247,
-            0.378928,
-            -0.902866,
-            0.202645,
-            -1.120171,
-            0.548113,
-            0.225239,
-            0.155128,
-            -0.719004,
-            -0.088457,
-            -0.047934,
-            -0.290179,
-            -0.713527,
-            0.109485,
-            -0.058881,
-            0.197997,
-            0.782527,
-            -0.340713,
-            0.402086,
-            0.617989,
-            -0.644194,
-            0.212122,
-            -0.178868,
-            0.450748,
-            -0.34007,
-            0.700888,
-            -0.11438,
-            -0.437129,
-            0.876918,
-            0.115418,
-            0.400021,
-            0.38345,
-            0.228783,
-            -0.712939,
-            1.033931,
-            0.019141,
-            0.234295,
-            -0.641412,
-            0.347562,
-            0.281232,
-            -1.025538,
-            -0.565528,
-            0.199486,
-            -0.523468,
-            -0.048982,
-            0.090189,
-            0.471387,
-            -0.122239,
-            -0.342236,
-            -0.716396,
-            -1.235716,
-            -0.16003,
-            0.113693,
-            0.671339,
-            -0.205079,
-            0.019947,
-            -0.585409,
-            -0.246022,
-            -1.003072,
-            1.76363,
-            0.567343,
-            -0.369613,
-            0.613968,
-            -0.657497,
-            -0.006477,
-            -1.256819,
-            -0.555521
-          ]
-        },
-        "n_layers": 24,
-        "hidden_size": 1536
-      },
-      "status": "scored",
-      "kl_global_avg": 1.663056,
-      "kl_per_prompt": [
-        1.443617,
-        1.933582,
-        1.645655,
-        1.78915,
-        1.954622,
-        2.221651,
-        3.135505,
-        1.31718,
-        0.255994,
-        0.933601
-      ],
-      "prompts_scored": 10,
-      "effective_total": 10,
-      "prompts_total": 10,
-      "scoring_time": 1.7,
-      "pre_scoring_time": 560.7,
-      "load_time": 67.3,
-      "early_stopped": false,
-      "early_stop_reason": null,
-      "shadow_topk": {
-        "kl_conditional_top128": 1.428012,
-        "student_top128_mass": 0.829586,
-        "tail_mass_gap_top128": 0.164322,
-        "teacher_top128_mass": 0.992609
-      },
-      "length_axis": {
-        "student_mean_gen": 54.5,
-        "teacher_mean_gen": 6.0,
-        "ratio": 9.083,
-        "penalty": 0.22,
-        "too_short_penalty": 1.0,
-        "too_long_penalty": 0.22,
-        "min_ratio": 0.08,
-        "too_short": false,
-        "source": "chat_probe"
-      },
-      "on_policy_rollouts_meta": {
-        "n_rollouts": 12,
-        "mean_gen_len": 124.8,
-        "collected_at": 161.5
-      },
-      "math_bench": {
-        "n": 16,
-        "correct": 1,
-        "pass_frac": 0.062,
-        "wall_s": 9.2,
-        "items": [
-          {
-            "src": "procedural_math/gcd_lcm",
-            "pred": "1",
-            "gold": "1",
-            "ok": true,
-            "gen_tokens": 4,
-            "tail": "#### 1"
-          },
-          {
-            "src": "procedural_math/mixture",
-            "pred": "4200",
-            "gold": "6000",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "#### 4200"
-          },
-          {
-            "src": "procedural_math/factorial_mod",
-            "pred": "17",
-            "gold": "6",
-            "ok": false,
-            "gen_tokens": 5,
-            "tail": "#### 17"
-          },
-          {
-            "src": "procedural_math/probability_count",
-            "pred": "10",
-            "gold": "35",
-            "ok": false,
-            "gen_tokens": 5,
-            "tail": "#### 10"
-          },
-          {
-            "src": "procedural_math/geometric_series",
-            "pred": "1008",
-            "gold": "1275",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "#### 1008"
-          },
-          {
-            "src": "procedural_math/digit_sum",
-            "pred": "122",
-            "gold": "36",
-            "ok": false,
-            "gen_tokens": 6,
-            "tail": "#### 122"
-          },
-          {
-            "src": "procedural_math/coin_change",
-            "pred": "122",
-            "gold": "3",
-            "ok": false,
-            "gen_tokens": 6,
-            "tail": "#### 122"
-          },
-          {
-            "src": "procedural_math/simultaneous",
-            "pred": "-4",
-            "gold": "3",
-            "ok": false,
-            "gen_tokens": 4,
-            "tail": "#### -4"
-          },
-          {
-            "src": "procedural_math/set_intersect",
-            "pred": "100",
-            "gold": "95",
-            "ok": false,
-            "gen_tokens": 6,
-            "tail": "#### 100"
-          },
-          {
-            "src": "procedural_math/unit_conversion",
-            "pred": "122",
-            "gold": "171",
-            "ok": false,
-            "gen_tokens": 6,
-            "tail": "#### 122"
-          },
-          {
-            "src": "procedural_math/triangle_area",
-            "pred": "10",
-            "gold": "15",
-            "ok": false,
-            "gen_tokens": 5,
-            "tail": "#### 10"
-          },
-          {
-            "src": "procedural_math/modular_linear",
-            "pred": "10",
-            "gold": "16",
-            "ok": false,
-            "gen_tokens": 5,
-            "tail": "#### 10"
-          },
-          {
-            "src": "procedural_math/arithmetic_series",
-            "pred": "1225",
-            "gold": "2100",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "#### 1225"
-          },
-          {
-            "src": "procedural_math/proportion",
-            "pred": "1008",
-            "gold": "326",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "#### 1008"
-          },
-          {
-            "src": "procedural_math/rate_distance",
-            "pred": "138",
-            "gold": "32",
-            "ok": false,
-            "gen_tokens": 6,
-            "tail": "#### 138"
-          },
-          {
-            "src": "procedural_math/time_arithmetic",
-            "pred": "1111",
-            "gold": "241",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "#### 1111"
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 5.8,
-        "mean_gen_tokens_correct": 4.0
-      },
-      "code_bench": {
-        "n": 10,
-        "correct": 2,
-        "pass_frac": 0.2,
-        "wall_s": 60.7,
-        "items": [
-          {
-            "task_id": "v27/string_predicate/00",
-            "entry_point": "is_balanced",
-            "ok": false,
-            "gen_tokens": 56,
-            "reason": "fail:IndentationError: unexpected indent",
-            "tail": "          stack.append(char)\n        elif char == ')':\n            if not stack:\n                return False\n            stack.pop()\n    return len(stack) == 0"
-          },
-          {
-            "task_id": "v27/pair_count/01",
-            "entry_point": "pair_count",
-            "ok": false,
-            "gen_tokens": 58,
-            "reason": "fail:IndentationError: unexpected indent",
-            "tail": "ount = 0\n    n = len(xs)\n    for i in range(n):\n        for j in range(i + 1, n):\n            if xs[i] + xs[j] == 4:\n                count += 1\n    return count"
-          },
-          {
-            "task_id": "v27/digit_sum/02",
-            "entry_point": "digital_root",
-            "ok": false,
-            "gen_tokens": 28,
-            "reason": "fail:IndentationError: unindent does not match any outer indentation level",
-            "tail": "    while n >= 10:\n        n = sum(int(digit) for digit in str(n))\n    return n"
-          },
-          {
-            "task_id": "v27/transform_list/03",
-            "entry_point": "transform",
-            "ok": false,
-            "gen_tokens": 13,
-            "reason": "fail:SyntaxError: 'return' outside function",
-            "tail": "    return [x + 3 for x in xs]"
-          },
-          {
-            "task_id": "v27/aggregate_list/04",
-            "entry_point": "aggregate",
-            "ok": false,
-            "gen_tokens": 16,
-            "reason": "fail:SyntaxError: 'return' outside function",
-            "tail": "    return sum(1 for x in xs if x > 0)"
-          },
-          {
-            "task_id": "v27/string_transform/05",
-            "entry_point": "alternate_case",
-            "ok": true,
-            "gen_tokens": 74,
-            "reason": "pass",
-            "tail": "sult.append(char.upper())\n            else:\n                result.append(char.lower())\n        else:\n            result.append(char)\n    return ''.join(result)"
-          },
-          {
-            "task_id": "v27/window_sum/06",
-            "entry_point": "window_max_sum",
-            "ok": true,
-            "gen_tokens": 146,
-            "reason": "pass",
-            "tail": "for i in range(k, len(xs)):\n        current_sum += xs[i] - xs[i - k]\n        if current_sum > max_sum:\n            max_sum = current_sum\n    \n    return max_sum"
-          },
-          {
-            "task_id": "v27/run_length/07",
-            "entry_point": "rle",
-            "ok": false,
-            "gen_tokens": 94,
-            "reason": "fail:IndentationError: unindent does not match any outer indentation level",
-            "tail": "           result.append((current_char, count))\n            current_char = char\n            count = 1\n    result.append((current_char, count))\n    return result"
-          },
-          {
-            "task_id": "v27/string_predicate/08",
-            "entry_point": "is_balanced",
-            "ok": false,
-            "gen_tokens": 56,
-            "reason": "fail:IndentationError: unexpected indent",
-            "tail": "          stack.append(char)\n        elif char == ')':\n            if not stack:\n                return False\n            stack.pop()\n    return len(stack) == 0"
-          },
-          {
-            "task_id": "v27/pair_count/09",
-            "entry_point": "pair_count",
-            "ok": false,
-            "gen_tokens": 58,
-            "reason": "fail:IndentationError: unexpected indent",
-            "tail": "ount = 0\n    n = len(xs)\n    for i in range(n):\n        for j in range(i + 1, n):\n            if xs[i] + xs[j] == 9:\n                count += 1\n    return count"
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 59.9,
-        "mean_gen_tokens_correct": 110.0
-      },
-      "reasoning_bench": {
-        "n": 12,
-        "correct": 6,
-        "pass_frac": 0.5,
-        "wall_s": 3.8,
-        "items": [
-          {
-            "src": "procedural_reasoning/sequence_next",
-            "pred": "(B)",
-            "gold": "(B)",
-            "ok": true,
-            "gen_tokens": 3,
-            "tail": "(B)"
-          },
-          {
-            "src": "procedural_reasoning/boolean_eval",
-            "pred": "(A)",
-            "gold": "(A)",
-            "ok": true,
-            "gen_tokens": 3,
-            "tail": "(A)"
-          },
-          {
-            "src": "procedural_reasoning/deduction",
-            "pred": "(B)",
-            "gold": "(A)",
-            "ok": false,
-            "gen_tokens": 3,
-            "tail": "(B)"
-          },
-          {
-            "src": "procedural_reasoning/analogy_letter",
-            "pred": "(B)",
-            "gold": "(B)",
-            "ok": true,
-            "gen_tokens": 3,
-            "tail": "(B)"
-          },
-          {
-            "src": "procedural_reasoning/odd_one_out",
-            "pred": "(B)",
-            "gold": "(A)",
-            "ok": false,
-            "gen_tokens": 3,
-            "tail": "(B)"
-          },
-          {
-            "src": "procedural_reasoning/ordering",
-            "pred": "(C)",
-            "gold": "(D)",
-            "ok": false,
-            "gen_tokens": 3,
-            "tail": "(C)"
-          },
-          {
-            "src": "procedural_reasoning/sequence_next",
-            "pred": "(B)",
-            "gold": "(A)",
-            "ok": false,
-            "gen_tokens": 3,
-            "tail": "(B)"
-          },
-          {
-            "src": "procedural_reasoning/boolean_eval",
-            "pred": "(A)",
-            "gold": "(A)",
-            "ok": true,
-            "gen_tokens": 3,
-            "tail": "(A)"
-          },
-          {
-            "src": "procedural_reasoning/deduction",
-            "pred": "(B)",
-            "gold": "(B)",
-            "ok": true,
-            "gen_tokens": 3,
-            "tail": "(B)"
-          },
-          {
-            "src": "procedural_reasoning/analogy_letter",
-            "pred": "(A)",
-            "gold": "(D)",
-            "ok": false,
-            "gen_tokens": 3,
-            "tail": "(A)"
-          },
-          {
-            "src": "procedural_reasoning/odd_one_out",
-            "pred": "(A)",
-            "gold": "(C)",
-            "ok": false,
-            "gen_tokens": 3,
-            "tail": "(A)"
-          },
-          {
-            "src": "procedural_reasoning/ordering",
-            "pred": "(B)",
-            "gold": "(B)",
-            "ok": true,
-            "gen_tokens": 3,
-            "tail": "(B)"
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 3.0,
-        "mean_gen_tokens_correct": 3.0
-      },
-      "knowledge_bench": {
-        "n": 0,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 0.0,
-        "items": [],
-        "error": null,
-        "mean_gen_tokens": 0.0,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "ifeval_bench": {
-        "n": 10,
-        "correct": 7,
-        "pass_frac": 0.7,
-        "wall_s": 106.2,
-        "items": [
-          {
-            "src": "procedural_ifeval/json_format",
-            "instruction_ids": [
-              "detectable_format:json_format"
-            ],
-            "per_instruction": [
-              true
-            ],
-            "ok": true,
-            "gen_tokens": 96,
-            "tail": "y spaces.\",\n  \"keywords\": [\n    \"library\",\n    \"renovation\",\n    \"reopening\",\n    \"community\",\n    \"modernization\"\n  ]\n}"
-          },
-          {
-            "src": "procedural_ifeval/max_words",
-            "instruction_ids": [
-              "length_constraints:number_words"
-            ],
-            "per_instruction": [
-              true
-            ],
-            "ok": true,
-            "gen_tokens": 17,
-            "tail": "Urban gardening brings fresh food, stress relief, and community connection to city living."
-          },
-          {
-            "src": "procedural_ifeval/min_words",
-            "instruction_ids": [
-              "length_constraints:number_words"
-            ],
-            "per_instruction": [
-              false
-            ],
-            "ok": false,
-            "gen_tokens": 94,
-            "tail": "rant hub for learning and connection, ensuring it remains a cherished cornerstone of the neighborhood for years to come."
-          },
-          {
-            "src": "procedural_ifeval/exact_sentences",
-            "instruction_ids": [
-              "length_constraints:number_sentences"
-            ],
-            "per_instruction": [
-              true
-            ],
-            "ok": true,
-            "gen_tokens": 31,
-            "tail": "raffic through treacherous waters. Their isolation demanded immense resilience and a deep dedication to maritime safety."
-          },
-          {
-            "src": "procedural_ifeval/ends_with_phrase",
-            "instruction_ids": [
-              "startend:end_checker"
-            ],
-            "per_instruction": [
-              false
-            ],
-            "ok": false,
-            "gen_tokens": 86,
-            "tail": "requently associate with a single generation of genetic memory and environmental cues rather than individual experience."
-          },
-          {
-            "src": "procedural_ifeval/all_lowercase",
-            "instruction_ids": [
-              "change_case:english_lowercase"
-            ],
-            "per_instruction": [
-              true
-            ],
-            "ok": true,
-            "gen_tokens": 53,
-            "tail": "ures are meticulously designed to withstand harsh weather conditions and seismic activity in high-altitude environments."
-          },
-          {
-            "src": "procedural_ifeval/bullet_list",
-            "instruction_ids": [
-              "detectable_format:number_bullet_lists"
-            ],
-            "per_instruction": [
-              false
-            ],
-            "ok": false,
-            "gen_tokens": 238,
-            "tail": " of the sun, Earth's magnetic field, and visual landmarks to find their way to the same specific groves year after year."
-          },
-          {
-            "src": "procedural_ifeval/no_comma",
-            "instruction_ids": [
-              "punctuation:no_comma"
-            ],
-            "per_instruction": [
-              true
-            ],
-            "ok": true,
-            "gen_tokens": 47,
-            "tail": "es mental health. Communities bond over shared green spaces. This hobby connects people with nature in concrete jungles."
-          },
-          {
-            "src": "procedural_ifeval/include_keyword",
-            "instruction_ids": [
-              "keywords:frequency"
-            ],
-            "per_instruction": [
-              true
-            ],
-            "ok": true,
-            "gen_tokens": 133,
-            "tail": "preserving this delicate ecological **compass**, ensuring that the ancient pathways remain intact for future migrations."
-          },
-          {
-            "src": "procedural_ifeval/all_uppercase",
-            "instruction_ids": [
-              "change_case:english_capital"
-            ],
-            "per_instruction": [
-              true
-            ],
-            "ok": true,
-            "gen_tokens": 256,
-            "tail": "GISTS ANALOGEOROLOGISTS ANALOGEOROLOGISTS ANALOGEOROLOGISTS ANALOGEOROLOGISTS ANALOGEOROLOGISTS ANALOGEOROLOGISTS ANALOG"
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 105.1,
-        "mean_gen_tokens_correct": 90.4
-      },
-      "aime_bench": {
-        "n": 10,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 5.2,
-        "items": [
-          {
-            "src": "procedural_aime/factor_chain",
-            "pred": "44",
-            "gold": "37",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{44}"
-          },
-          {
-            "src": "procedural_aime/chained_modular",
-            "pred": "10",
-            "gold": "22",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{10}"
-          },
-          {
-            "src": "procedural_aime/lcm_residue",
-            "pred": "105",
-            "gold": "65",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{105}"
-          },
-          {
-            "src": "procedural_aime/iterated_digit",
-            "pred": "108",
-            "gold": "59",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{108}"
-          },
-          {
-            "src": "procedural_aime/diophantine",
-            "pred": "10",
-            "gold": "8",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{10}"
-          },
-          {
-            "src": "procedural_aime/factor_chain",
-            "pred": "44",
-            "gold": "45",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{44}"
-          },
-          {
-            "src": "procedural_aime/chained_modular",
-            "pred": "10",
-            "gold": "50",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{10}"
-          },
-          {
-            "src": "procedural_aime/lcm_residue",
-            "pred": "105",
-            "gold": "83",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{105}"
-          },
-          {
-            "src": "procedural_aime/iterated_digit",
-            "pred": "108",
-            "gold": "73",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{108}"
-          },
-          {
-            "src": "procedural_aime/diophantine",
-            "pred": "10",
-            "gold": "202",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{10}"
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 7.4,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "mbpp_bench": {
-        "n": 10,
-        "correct": 10,
-        "pass_frac": 1.0,
-        "wall_s": 90.9,
-        "items": [
-          {
-            "task_id": "v27/string_predicate/00",
-            "entry_point": "is_anagram",
-            "ok": true,
-            "gen_tokens": 64,
-            "reason": "pass",
-            "tail": "nagram of string ``b`` (case-sensitive). Different lengths return False.\n    \"\"\"\n    if len(a) != len(b):\n        return False\n    return sorted(a) == sorted(b)"
-          },
-          {
-            "task_id": "v27/string_transform/01",
-            "entry_point": "swap_pairs",
-            "ok": true,
-            "gen_tokens": 106,
-            "reason": "pass",
-            "tail": "n place.\n    \"\"\"\n    chars = list(s)\n    for i in range(0, len(chars) - 1, 2):\n        chars[i], chars[i + 1] = chars[i + 1], chars[i]\n    return ''.join(chars)"
-          },
-          {
-            "task_id": "v27/aggregate_list/02",
-            "entry_point": "aggregate",
-            "ok": true,
-            "gen_tokens": 45,
-            "reason": "pass",
-            "tail": "   \"\"\"Return the difference between the largest and the smallest value in ``xs``. The list always has at least one element.\n    \"\"\"\n    return max(xs) - min(xs)"
-          },
-          {
-            "task_id": "v27/run_length/03",
-            "entry_point": "rle",
-            "ok": true,
-            "gen_tokens": 100,
-            "reason": "pass",
-            "tail": "           result.append((current_char, count))\n            current_char = char\n            count = 1\n    result.append((current_char, count))\n    return result"
-          },
-          {
-            "task_id": "v27/transform_list/04",
-            "entry_point": "transform",
-            "ok": true,
-            "gen_tokens": 18,
-            "reason": "pass",
-            "tail": "def transform(xs):\n    return [x + 3 for x in xs]"
-          },
-          {
-            "task_id": "v27/digit_sum/05",
-            "entry_point": "count_divisors",
-            "ok": true,
-            "gen_tokens": 103,
-            "reason": "pass",
-            "tail": " <= n:\n        if n % i == 0:\n            if i * i == n:\n                count += 1\n            else:\n                count += 2\n        i += 1\n    return count"
-          },
-          {
-            "task_id": "v27/window_sum/06",
-            "entry_point": "window_max_sum",
-            "ok": true,
-            "gen_tokens": 191,
-            "reason": "pass",
-            "tail": "):\n        current_sum = current_sum - xs[i - 1] + xs[i + k - 1]\n        if current_sum > max_sum:\n            max_sum = current_sum\n    \n    return max_sum\n```"
-          },
-          {
-            "task_id": "v27/pair_count/07",
-            "entry_point": "pair_count",
-            "ok": true,
-            "gen_tokens": 100,
-            "reason": "pass",
-            "tail": "ount = 0\n    n = len(xs)\n    for i in range(n):\n        for j in range(i + 1, n):\n            if xs[i] + xs[j] == 4:\n                count += 1\n    return count"
-          },
-          {
-            "task_id": "v27/string_predicate/08",
-            "entry_point": "is_anagram",
-            "ok": true,
-            "gen_tokens": 64,
-            "reason": "pass",
-            "tail": "nagram of string ``b`` (case-sensitive). Different lengths return False.\n    \"\"\"\n    if len(a) != len(b):\n        return False\n    return sorted(a) == sorted(b)"
-          },
-          {
-            "task_id": "v27/string_transform/09",
-            "entry_point": "swap_pairs",
-            "ok": true,
-            "gen_tokens": 106,
-            "reason": "pass",
-            "tail": "n place.\n    \"\"\"\n    chars = list(s)\n    for i in range(0, len(chars) - 1, 2):\n        chars[i], chars[i + 1] = chars[i + 1], chars[i]\n    return ''.join(chars)"
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 89.7,
-        "mean_gen_tokens_correct": 89.7
-      },
-      "tool_use_bench": {
-        "n": 8,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 6.2,
-        "items": [
-          {
-            "src": "procedural_math/probability_count",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "10",
-            "gold": "16",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{10}"
-          },
-          {
-            "src": "procedural_math/triangle_area",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "155",
-            "gold": "45",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{155}"
-          },
-          {
-            "src": "procedural_math/unit_conversion",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "288",
-            "gold": "208",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{288}"
-          },
-          {
-            "src": "procedural_math/factorial_mod",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "13",
-            "gold": "6",
-            "ok": false,
-            "gen_tokens": 7,
-            "tail": "\\boxed{13}"
-          },
-          {
-            "src": "procedural_math/modular_linear",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "1",
-            "gold": "11",
-            "ok": false,
-            "gen_tokens": 6,
-            "tail": "\\boxed{1}"
-          },
-          {
-            "src": "procedural_math/coin_change",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "105",
-            "gold": "6",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{105}"
-          },
-          {
-            "src": "procedural_math/mixture",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "3250",
-            "gold": "4980",
-            "ok": false,
-            "gen_tokens": 9,
-            "tail": "\\boxed{3250}"
-          },
-          {
-            "src": "procedural_math/time_arithmetic",
-            "tool_used": false,
-            "tool_result": null,
-            "pred": "101",
-            "gold": "609",
-            "ok": false,
-            "gen_tokens": 8,
-            "tail": "\\boxed{101}"
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 7.6,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "self_consistency_bench": {
-        "n": 0,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 0.0,
-        "items": [],
-        "error": null,
-        "mean_gen_tokens": 0.0,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "arc_bench": {
-        "n": 0,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 0.0,
-        "items": [],
-        "error": null,
-        "mean_gen_tokens": 0.0,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "truthful_bench": {
-        "n": 0,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 0.0,
-        "items": [],
-        "error": null,
-        "mean_gen_tokens": 0.0,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "long_context_bench": {
-        "n": 6,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 3.1,
-        "items": [
-          {
-            "src": "long_context",
-            "gold": "XWMBUM7",
-            "pred_tail": "D3K64",
-            "ok": false,
-            "gen_tokens": 6,
-            "needle_position": 19,
-            "confuser_positions": [
-              1,
-              42,
-              23
-            ],
-            "confuser_hit": false,
-            "gold_in_pred": false
-          },
-          {
-            "src": "long_context",
-            "gold": "N2NERPB",
-            "pred_tail": "256J4U",
-            "ok": false,
-            "gen_tokens": 7,
-            "needle_position": 13,
-            "confuser_positions": [
-              9,
-              35,
-              30
-            ],
-            "confuser_hit": false,
-            "gold_in_pred": false
-          },
-          {
-            "src": "long_context",
-            "gold": "V8C3MQG",
-            "pred_tail": "7YQMBB",
-            "ok": false,
-            "gen_tokens": 6,
-            "needle_position": 15,
-            "confuser_positions": [
-              42,
-              7,
-              18
-            ],
-            "confuser_hit": false,
-            "gold_in_pred": false
-          },
-          {
-            "src": "long_context",
-            "gold": "7PXTYV2",
-            "pred_tail": "7S2P2B",
-            "ok": false,
-            "gen_tokens": 7,
-            "needle_position": 26,
-            "confuser_positions": [
-              1,
-              15,
-              29
-            ],
-            "confuser_hit": false,
-            "gold_in_pred": false
-          },
-          {
-            "src": "long_context",
-            "gold": "K4USQ2C",
-            "pred_tail": "9RZC62",
-            "ok": false,
-            "gen_tokens": 7,
-            "needle_position": 13,
-            "confuser_positions": [
-              3,
-              33,
-              20
-            ],
-            "confuser_hit": false,
-            "gold_in_pred": false
-          },
-          {
-            "src": "long_context",
-            "gold": "ERYJ6SQ",
-            "pred_tail": "FJNBK6",
-            "ok": false,
-            "gen_tokens": 6,
-            "needle_position": 12,
-            "confuser_positions": [
-              19,
-              22,
-              25
-            ],
-            "confuser_hit": false,
-            "gold_in_pred": false
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 6.5,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "procedural_bench": {
-        "n": 0,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 0.0,
-        "items": [],
-        "error": null,
-        "mean_gen_tokens": 0.0,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "robustness_bench": {
-        "n": 16,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 11.2,
-        "items": [
-          {
-            "src": "procedural_math/geometric_series",
-            "perturbation": "instruction_synonym",
-            "pred": "100",
-            "gold": "75",
-            "ok": false,
-            "gen_tokens": 6
-          },
-          {
-            "src": "procedural_math/geometric_series",
-            "perturbation": "context_prefix",
-            "pred": "100",
-            "gold": "75",
-            "ok": false,
-            "gen_tokens": 6
-          },
-          {
-            "src": "procedural_math/mixture",
-            "perturbation": "instruction_synonym",
-            "pred": "4200",
-            "gold": "5590",
-            "ok": false,
-            "gen_tokens": 7
-          },
-          {
-            "src": "procedural_math/mixture",
-            "perturbation": "context_prefix",
-            "pred": "4200",
-            "gold": "5590",
-            "ok": false,
-            "gen_tokens": 7
-          },
-          {
-            "src": "procedural_math/probability_count",
-            "perturbation": "instruction_synonym",
-            "pred": "10",
-            "gold": "25",
-            "ok": false,
-            "gen_tokens": 5
-          },
-          {
-            "src": "procedural_math/probability_count",
-            "perturbation": "context_prefix",
-            "pred": "10",
-            "gold": "25",
-            "ok": false,
-            "gen_tokens": 5
-          },
-          {
-            "src": "procedural_math/polynomial_eval",
-            "perturbation": "instruction_synonym",
-            "pred": "1",
-            "gold": "28",
-            "ok": false,
-            "gen_tokens": 18
-          },
-          {
-            "src": "procedural_math/polynomial_eval",
-            "perturbation": "context_prefix",
-            "pred": "10",
-            "gold": "28",
-            "ok": false,
-            "gen_tokens": 5
-          },
-          {
-            "src": "procedural_math/proportion",
-            "perturbation": "instruction_synonym",
-            "pred": "1084",
-            "gold": "1287",
-            "ok": false,
-            "gen_tokens": 7
-          },
-          {
-            "src": "procedural_math/proportion",
-            "perturbation": "context_prefix",
-            "pred": "1021",
-            "gold": "1287",
-            "ok": false,
-            "gen_tokens": 7
-          },
-          {
-            "src": "procedural_math/set_intersect",
-            "perturbation": "instruction_synonym",
-            "pred": "100",
-            "gold": "103",
-            "ok": false,
-            "gen_tokens": 6
-          },
-          {
-            "src": "procedural_math/set_intersect",
-            "perturbation": "context_prefix",
-            "pred": "100",
-            "gold": "103",
-            "ok": false,
-            "gen_tokens": 6
-          },
-          {
-            "src": "procedural_math/rate_distance",
-            "perturbation": "instruction_synonym",
-            "pred": "102",
-            "gold": "28",
-            "ok": false,
-            "gen_tokens": 6
-          },
-          {
-            "src": "procedural_math/rate_distance",
-            "perturbation": "context_prefix",
-            "pred": "102",
-            "gold": "28",
-            "ok": false,
-            "gen_tokens": 6
-          },
-          {
-            "src": "procedural_math/digit_sum",
-            "perturbation": "instruction_synonym",
-            "pred": "122",
-            "gold": "167",
-            "ok": false,
-            "gen_tokens": 6
-          },
-          {
-            "src": "procedural_math/digit_sum",
-            "perturbation": "context_prefix",
-            "pred": "113",
-            "gold": "167",
-            "ok": false,
-            "gen_tokens": 6
-          }
-        ],
-        "error": null,
-        "mean_gen_tokens": 6.8,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "noise_resistance_bench": {
-        "n": 0,
-        "correct": 0,
-        "pass_frac": 0.0,
-        "wall_s": 0.0,
-        "items": [],
-        "error": null,
-        "mean_gen_tokens": 0.0,
-        "mean_gen_tokens_correct": 0.0
-      },
-      "bench_gate": {
-        "run": true,
-        "reason": "no_king_kl",
-        "only_kl_candidates": true,
-        "kl_candidate_slack": 0.0,
-        "student_kl": 1.663056,
-        "king_kl": null,
-        "wall_s": 296.5
-      },
-      "on_policy_rkl": {
-        "n_rollouts": 12,
-        "tokens": 1497,
-        "mean_rkl": 0.520277,
-        "mean_fkl": 1.152991,
-        "mean_skl": 0.275421,
-        "mean_sampled_gap": 0.594948,
-        "mean_gen_len": 124.8,
-        "skew_alpha": 0.1,
-        "top_k": 128,
-        "per_rollout": [
-          {
-            "prompt": "Who painted the Mona Lisa?",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.409653,
-            "fkl": 0.928422,
-            "skl": 0.210175,
-            "sampled_gap": 0.44171,
-            "tail": " period: Early 16th century (around 1503\u20131506, possibly continued until 1517).\n    *   Current location: The Louvre Museum, Paris.\n\n3.  **Formulate the answer:** Directly state the name of the artist."
-          },
-          {
-            "prompt": "How many continents are there?",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.582701,
-            "fkl": 1.658169,
-            "skl": 0.383538,
-            "sampled_gap": 0.757408,
-            "tail": "frica, Antarctica, Asia, Europe, North America, Oceania/Australia, South America).\n    *   **Alternative Answers (6 continents):** The \"6-continent model\" often merges Europe and Asia into Eurasia, or"
-          },
-          {
-            "prompt": "Explain a sunset using exactly three adjectives.",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.621109,
-            "fkl": 1.525348,
-            "skl": 0.362313,
-            "sampled_gap": 0.718639,
-            "tail": "m, soft.\n    *   Mood/Quality: Beautiful, breathtaking, dramatic, serene, fading, fading, brilliant, melancholic.\n    *   State: Setting, dying, sinking, glowing.\n\n3.  **Drafting Sentences (aiming for"
-          },
-          {
-            "prompt": "What is the square root of 144?",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.429711,
-            "fkl": 0.696605,
-            "skl": 0.179877,
-            "sampled_gap": 0.384291,
-            "tail": "eed to find a number that, when multiplied by itself, equals 144.\n3.  **Calculate:**\n    *   $10 \\times 10 = 100$ (too small)\n    *   $12 \\times 12 = 144$ (correct)\n    *   Also, $\\sqrt{144} = 12$.\n4."
-          },
-          {
-            "prompt": "Translate to French: The cat sat on the mat.",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.611311,
-            "fkl": 1.10299,
-            "skl": 0.276324,
-            "sampled_gap": 0.677515,
-            "tail": "e chat\"\n    *   \"sat\" -> \"s'\u00e9tait assis\" (pass\u00e9 compos\u00e9 would be \"s'est assis\", but simple past \"pass\u00e9 simple\" is \"s'assit\", though in modern French narrative, \"\u00e9tait assis\" or \"s'est assis\" is common"
-          },
-          {
-            "prompt": "In plain language, what is an operating system?",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.552007,
-            "fkl": 1.453894,
-            "skl": 0.328995,
-            "sampled_gap": 0.621598,
-            "tail": "fy Key Concepts:**\n    *   What does an OS do? It manages hardware and software.\n    *   What does the user see? Windows, macOS, iOS, Android icons.\n    *   What does the computer do? Processes tasks,"
-          },
-          {
-            "prompt": "In exactly one sentence, outline what courage is.",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.690625,
-            "fkl": 1.588047,
-            "skl": 0.396777,
-            "sampled_gap": 0.723275,
-            "tail": "olved. It's about acting in spite of fear. It involves strength, bravery, moral fortitude, and taking action.\n\n3.  **Drafting Sentences:**\n    *   *Draft 1:* Courage is the ability to face fear and do"
-          },
-          {
-            "prompt": "Outline what a compiler does, in one paragraph.",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.454211,
-            "fkl": 1.035425,
-            "skl": 0.244046,
-            "sampled_gap": 0.524094,
-            "tail": "binary, machine-readable, low-level).\n    *   Process: Translation, parsing, optimization, code generation.\n    *   Timing: Before execution (static translation).\n    *   Purpose: Bridge between human"
-          },
-          {
-            "prompt": "What is a linked enumerate in computer science? Two sentences.",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.589372,
-            "fkl": 1.265752,
-            "skl": 0.276325,
-            "sampled_gap": 0.690847,
-            "tail": "ist, Enumeration (enum), Enumerate (function), Linked enumerate?\n    *   Hypothesis: It's likely a confusion or a non-standard term.\n        *   Possibility A: \"Linked List\" + \"Enumeration\".\n        *"
-          },
-          {
-            "prompt": "Give three synonyms of 'happy', one per line.",
-            "gen_len": 123,
-            "tokens_scored": 123,
-            "rkl": 0.326833,
-            "fkl": 0.623001,
-            "skl": 0.162384,
-            "sampled_gap": 0.470018,
-            "tail": "   *   Joyful\n    *   Glad\n    *   Content\n    *   Cheery\n    *   Blissful\n    *   Pleased\n    *   Delighted\n    *   Cheerful\n\n3.  **Select Three Distinct Synonyms:**\n    *   Joyful\n    *   Glad\n    *"
-          },
-          {
-            "prompt": "Translate to Dutch: The red house is on the left.",
-            "gen_len": 125,
-            "tokens_scored": 125,
-            "rkl": 0.430915,
-            "fkl": 0.787817,
-            "skl": 0.202737,
-            "sampled_gap": 0.507985,
-            "tail": "  \"The\": Definite article (singular, neuter/feminine/masculine depending on noun).\n    *   \"red\": Adjective (color).\n    *   \"house\": Noun (gender: het huis).\n    *   \"is\": Verb (to be, present tense,"
-          },
-          {
-            "prompt": "Define machine learning in one sentence.",
-            "gen_len": 124,
-            "tokens_scored": 124,
-            "rkl": 0.541953,
-            "fkl": 1.162009,
-            "skl": 0.279784,
-            "sampled_gap": 0.620198,
-            "tail": "rogramming for every task.\n    *   Improvement over time (usually).\n\n3.  **Draft Initial Definitions:**\n    *   Machine learning is a subset of artificial intelligence where computers learn from data."
-          }
-        ],
-        "scoring_time": 5.0
-      },
-      "judge_probe": {
-        "n": 16,
-        "n_valid": 16,
-        "mean_score": 4.875,
-        "normalized": 0.9688,
-        "per_prompt": [
-          {
-            "prompt": "Name one thing you should check before submitting a pull request. Single-line answer.",
-            "response_preview": "Run the project's tests to ensure nothing is broken.",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "In one short paragraph, explain why you would or wouldn't suggest ice baths for sore muscles after exercise.",
-            "response_preview": "I would suggest ice baths with caution, as they are effective at rapidly reducing inflammation and pain immediately afte",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "You have a 3-liter jug and a 5-liter jug and a water source. Outline briefly how to measure exactly 4 liters.",
-            "response_preview": "Here is the step-by-step process to measure exactly 4 liters using a 3-liter jug and a 5-liter jug:\n\n1.  **Fill** the 3-",
-            "raw": "4",
-            "score": 4
-          },
-          {
-            "prompt": "Offer one creative metaphor comparing a commit history to something from everyday life.",
-            "response_preview": "A commit history is like the **layered strata of a geological formation**, where each commit is a distinct sediment depo",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "If it is 2:30 PM now, what time will it be 250 minutes from now? Demonstrate your work.",
-            "response_preview": "To determine the time 250 minutes from 2:30 PM, we will break the calculation down into hours and minutes, then add them",
-            "raw": "4",
-            "score": 4
-          },
-          {
-            "prompt": "What is the output of `print(list(range(3, 10, 2)))` in Python? Just the output.",
-            "response_preview": "[3, 5, 7, 9]",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Produce a haiku about winter. The only output should be three lines of the haiku \u2014 no intro, no explanation.",
-            "response_preview": "Silent snow descends,\nBare branches hold the frozen air,\nSpring waits in the cold.",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Outline in one sentence why running database migrations in transactions is usually a good idea.",
-            "response_preview": "Running database migrations in transactions is usually a good idea because it ensures atomicity, guaranteeing that eithe",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Write a SQL query that returns the top 5 customers by total `amount` from a table `orders(customer_id, amount)`. Just the query.",
-            "response_preview": "```sql\nSELECT customer_id, SUM(amount) AS total_amount\nFROM orders\nGROUP BY customer_id\nORDER BY total_amount DESC\nLIMIT",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Concisely: what does a cache do in a CPU?",
-            "response_preview": "A CPU cache is a small, ultra-fast memory located directly on the processor that stores frequently accessed data and ins",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Demonstrate a simple Bash loop that prints numbers 1 through 5, one per line.",
-            "response_preview": "Here is a simple Bash loop that prints numbers from 1 to 5:\n\n```bash\nfor i in {1..5}; do\n    echo \"$i\"\ndone\n```\n\n**How i",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "When is it appropriate to ask for clarification when given an ambiguous task? One sentence.",
-            "response_preview": "It is appropriate to ask for clarification immediately upon recognizing ambiguity, as early intervention prevents wasted",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Write the opening two sentences of a short story set on a lighthouse during a storm.",
-            "response_preview": "The wind howled like a starving beast, tearing at the iron grating of the lantern room and threatening to unseat the gre",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Write a one-line Python enumerate comprehension that returns the squares of the even numbers from 0 to 20 inclusive.",
-            "response_preview": "`[x**2 for i, x in enumerate(range(21)) if x % 2 == 0]`",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "What's the difference between correlation and causation? Provide one example in 2 sentences.",
-            "response_preview": "Correlation describes a statistical relationship where two variables change together, while causation indicates that one",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "prompt": "Why does water expand when it freezes? Offer a concise explanation.",
-            "response_preview": "Water expands when it freezes primarily due to the unique behavior of **hydrogen bonds** between its molecules.\n\nIn liqu",
-            "raw": "5",
-            "score": 5
-          }
-        ],
-        "scoring_time": 27.4,
-        "in_composite": true,
-        "version": 1
-      },
-      "chat_turns_probe": {
-        "n": 6,
-        "n_valid": 6,
-        "n_turns": 3,
-        "mean_score": 4.667,
-        "normalized": 0.9167,
-        "per_convo": [
-          {
-            "seed": "I'm planning a small dinner party for 6 people. Can you help me pick a theme?",
-            "raw": "3",
-            "score": 3
-          },
-          {
-            "seed": "Outline the difference between `let`, `const`, and `var` in JavaScript.",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "seed": "My tomato plants have yellow leaves. What could be wrong?",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "seed": "Offer me a simple recipe for chocolate chip cookies.",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "seed": "Help me debug: my Python script prints 'Hello' twice even though I only called print once.",
-            "raw": "5",
-            "score": 5
-          },
-          {
-            "seed": "I want to meditate but can't focus. Any practical starting advice?",
-            "raw": "5",
-            "score": 5
-          }
-        ],
-        "scoring_time": 10.5,
-        "in_composite": true,
-        "version": 1
-      }
-    },
-    "Qwen/Qwen3.5-4B": {
-      "status": "teacher_sanity",
-      "capability": {
-        "n": 36,
-        "correct": 32,
-        "pass_frac": 0.889,
-        "teacher_pass_frac": 0.889,
-        "items": [],
-        "source": "teacher_sanity_probe"
-      }
-    }
-  },
-  "timings": {
-    "vllm_startup": 96.7,
-    "vllm_generation": 7.5,
-    "teacher_probe_refs": 8.1,
-    "teacher_logits_pass": 0.0,
-    "teacher_hf_load": 0.0,
-    "on_policy_rkl": 5.0,
-    "judge_probe": 27.4,
-    "chat_turns_probe": 10.5,
-    "phase_b_total": 45.9
-  }
-}