| { |
| "artifact": "fp8-block-calib-v1", |
| "repository": "ProCreations/Nanbeige4.3-33B-FP8", |
| "as_of": "2026-10-25T17:30:43.798727-04:00", |
| "publication_authorized": false, |
| "source_manifest_sha256": "5f3d0a025474f103d963541841d934f619b917f47d12fd547a3ed0f5d474ae98", |
| "source_revision": "c1093c501cc6460427c0ac634db7223d0450c0ee", |
| "calibration_manifest_sha256": "4305272fa895081ac90038f656cf3eec2e40cedf17edc3110e459304abc12191", |
| "eval_manifest_sha256": "4fe31e47a0b7426e1e3f89fa4945343561681f5112ef125cd43958cac3556fe5", |
| "official_assistant_kld": { |
| "corpus": "heldout-v1", |
| "estimator": "BF16 top-256 + aggregate tail, natural logarithm, full padded vocabulary", |
| "domains": { |
| "all": { |
| "n": 634268, |
| "mean_kld": 0.007367580809794595, |
| "p99_kld": 0.12034924847789354, |
| "top1_agreement": 0.9782410589845302, |
| "decision_n": 1523, |
| "decision_mean_kld": 0.004023066272527601, |
| "decision_p99_kld": 0.05248585987264853, |
| "decision_top1_agreement": 0.9809586342744583, |
| "full_validation": { |
| "n": 2176, |
| "full_mean": 0.021584294423247444, |
| "approx_mean": 0.02125973179784336, |
| "mean_underestimate": 0.00032456262540408624, |
| "p99_abs_error": 0.002764771261159961, |
| "max_abs_error": 0.24186852245590007 |
| }, |
| "all_position_mean_kld": 0.05796733690664629, |
| "all_position_p99_kld": 1.168679188344561, |
| "all_position_top1_agreement": 0.951873018144382, |
| "assistant_n": 634268, |
| "all_position_n": 1006317, |
| "prompt_n": 372049, |
| "prompt_mean_kld": 0.14422964671002683, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 213439, |
| "mean_kld": 0.008057215174137846, |
| "p99_kld": 0.139296367399638, |
| "top1_agreement": 0.9787901929825383 |
| }, |
| "assistant_thinking": { |
| "n": 396567, |
| "mean_kld": 0.007403651097538524, |
| "p99_kld": 0.11431641224602686, |
| "top1_agreement": 0.9768084585958993 |
| }, |
| "assistant_tool_call": { |
| "n": 24262, |
| "mean_kld": 0.0007111157660716952, |
| "p99_kld": 0.0122668142214743, |
| "top1_agreement": 0.9968263127524524 |
| }, |
| "forced_assistant_prefix": { |
| "n": 624, |
| "mean_kld": 0.0008443500722979389, |
| "p99_kld": 0.036135052808778455, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 166546, |
| "mean_kld": 0.13669373488755113, |
| "p99_kld": 2.7352043574177105, |
| "top1_agreement": 0.8955964118021448 |
| }, |
| "template_boundary": { |
| "n": 1420, |
| "mean_kld": 0.02594404540389698, |
| "p99_kld": 0.5411472371514969, |
| "top1_agreement": 0.9767605633802817 |
| }, |
| "tool_output": { |
| "n": 70463, |
| "mean_kld": 0.30416640099858205, |
| "p99_kld": 6.324727769738451, |
| "top1_agreement": 0.8740189886890992 |
| }, |
| "user_prompt": { |
| "n": 132996, |
| "mean_kld": 0.07086571420008837, |
| "p99_kld": 1.6006519708282312, |
| "top1_agreement": 0.9373514992932118 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 1461, |
| "full_mean": 0.004782547311488508, |
| "approx_mean": 0.004781964061061122, |
| "mean_underestimate": 5.832504273854853e-07, |
| "p99_abs_error": 9.295809699652282e-06, |
| "max_abs_error": 0.0003091040153110833 |
| }, |
| "full_validation_prompt": { |
| "n": 715, |
| "full_mean": 0.055916256004058376, |
| "approx_mean": 0.054929687970485114, |
| "mean_underestimate": 0.0009865680335732609, |
| "p99_abs_error": 0.011844877901196151, |
| "max_abs_error": 0.24186852245590007 |
| }, |
| "all_position_decision_mean_kld": 0.006487665287840157, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 362, |
| "mean_kld": 0.005601145813384146, |
| "p99_kld": 0.06973303093308376, |
| "top1_agreement": 0.9696132596685083 |
| }, |
| "eos": { |
| "n": 423, |
| "mean_kld": 0.0025081528407262, |
| "p99_kld": 0.029918783738657098, |
| "top1_agreement": 0.9881796690307328 |
| }, |
| "reverification_start": { |
| "n": 122, |
| "mean_kld": 0.008281744634107533, |
| "p99_kld": 0.05017633308086587, |
| "top1_agreement": 0.9590163934426229 |
| }, |
| "think_close": { |
| "n": 397, |
| "mean_kld": 0.005108800118109972, |
| "p99_kld": 0.07364685660738739, |
| "top1_agreement": 0.9798488664987406 |
| }, |
| "tool_call_close": { |
| "n": 219, |
| "mean_kld": 2.1627920153876327e-11, |
| "p99_kld": 2.898614009420552e-10, |
| "top1_agreement": 1.0 |
| } |
| } |
| }, |
| "agentic": { |
| "n": 70490, |
| "mean_kld": 0.0030562890478571902, |
| "p99_kld": 0.028789159059545794, |
| "top1_agreement": 0.9819832600368846, |
| "decision_n": 600, |
| "decision_mean_kld": 0.0023578799057667604, |
| "decision_p99_kld": 0.02795729858684846, |
| "decision_top1_agreement": 0.9816666666666667, |
| "full_validation": { |
| "n": 456, |
| "full_mean": 0.07565850656635484, |
| "approx_mean": 0.07420076055928274, |
| "mean_underestimate": 0.0014577460070720949, |
| "p99_abs_error": 0.01932041891701553, |
| "max_abs_error": 0.24186852245590007 |
| }, |
| "all_position_mean_kld": 0.13946728314091697, |
| "all_position_p99_kld": 3.2237079280215952, |
| "all_position_top1_agreement": 0.9120975208311902, |
| "assistant_n": 70490, |
| "all_position_n": 349956, |
| "prompt_n": 279466, |
| "prompt_mean_kld": 0.1738743701340388, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 14976, |
| "mean_kld": 0.0038130090757514975, |
| "p99_kld": 0.023476198348296045, |
| "top1_agreement": 0.9750934829059829 |
| }, |
| "assistant_thinking": { |
| "n": 34277, |
| "mean_kld": 0.004119720016701908, |
| "p99_kld": 0.0335689234217045, |
| "top1_agreement": 0.9760772529684628 |
| }, |
| "assistant_tool_call": { |
| "n": 21237, |
| "mean_kld": 0.000806260208716276, |
| "p99_kld": 0.014112304075845728, |
| "top1_agreement": 0.9963742524838725 |
| }, |
| "forced_assistant_prefix": { |
| "n": 130, |
| "mean_kld": 0.00044959917448331665, |
| "p99_kld": 0.007315506886369604, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 154661, |
| "mean_kld": 0.1440098008371929, |
| "p99_kld": 2.7978990170313462, |
| "top1_agreement": 0.8902244263259645 |
| }, |
| "template_boundary": { |
| "n": 563, |
| "mean_kld": 0.03297958036923673, |
| "p99_kld": 0.6891723849177426, |
| "top1_agreement": 0.9751332149200711 |
| }, |
| "tool_output": { |
| "n": 70463, |
| "mean_kld": 0.30416640099858205, |
| "p99_kld": 6.324727769738451, |
| "top1_agreement": 0.8740189886890992 |
| }, |
| "user_prompt": { |
| "n": 53649, |
| "mean_kld": 0.0907411480436643, |
| "p99_kld": 2.114305469422486, |
| "top1_agreement": 0.9324684523476673 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 104, |
| "full_mean": 0.0019695944648873303, |
| "approx_mean": 0.001965068470107848, |
| "mean_underestimate": 4.525994779482884e-06, |
| "p99_abs_error": 4.479286198386308e-05, |
| "max_abs_error": 0.0003091040153110833 |
| }, |
| "full_validation_prompt": { |
| "n": 352, |
| "full_mean": 0.09743023059633388, |
| "approx_mean": 0.0955431241310844, |
| "mean_underestimate": 0.0018871064652494572, |
| "p99_abs_error": 0.022445905016727778, |
| "max_abs_error": 0.24186852245590007 |
| }, |
| "all_position_decision_mean_kld": 0.008172344295016386, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 94, |
| "mean_kld": 0.004103680569481858, |
| "p99_kld": 0.014667860800303238, |
| "top1_agreement": 0.9361702127659575 |
| }, |
| "eos": { |
| "n": 155, |
| "mean_kld": 0.0021693042402696787, |
| "p99_kld": 0.028801755126075104, |
| "top1_agreement": 0.9870967741935484 |
| }, |
| "reverification_start": { |
| "n": 16, |
| "mean_kld": 0.009852361769401453, |
| "p99_kld": 0.04115630742252282, |
| "top1_agreement": 0.875 |
| }, |
| "think_close": { |
| "n": 145, |
| "mean_kld": 0.0036903587561386343, |
| "p99_kld": 0.02924118683569686, |
| "top1_agreement": 0.993103448275862 |
| }, |
| "tool_call_close": { |
| "n": 190, |
| "mean_kld": 2.4928612325481013e-11, |
| "p99_kld": 3.362777045454479e-10, |
| "top1_agreement": 1.0 |
| } |
| } |
| }, |
| "chat": { |
| "n": 112182, |
| "mean_kld": 0.01129191965548786, |
| "p99_kld": 0.1633397121356422, |
| "top1_agreement": 0.9700843272539267, |
| "decision_n": 197, |
| "decision_mean_kld": 0.006545786482879379, |
| "decision_p99_kld": 0.06570825165322552, |
| "decision_top1_agreement": 0.9796954314720813, |
| "full_validation": { |
| "n": 272, |
| "full_mean": 0.006739306846575501, |
| "approx_mean": 0.006721999377542803, |
| "mean_underestimate": 1.730746903269902e-05, |
| "p99_abs_error": 0.00013107368363489947, |
| "max_abs_error": 0.00032759181097957377 |
| }, |
| "all_position_mean_kld": 0.011778191859905739, |
| "all_position_p99_kld": 0.16555953515016872, |
| "all_position_top1_agreement": 0.9689836541106452, |
| "assistant_n": 112182, |
| "all_position_n": 119969, |
| "prompt_n": 7787, |
| "prompt_mean_kld": 0.018783583979593232, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 64198, |
| "mean_kld": 0.011823489979899907, |
| "p99_kld": 0.18022920956472085, |
| "top1_agreement": 0.9721798186859404 |
| }, |
| "assistant_thinking": { |
| "n": 47984, |
| "mean_kld": 0.010580729431942413, |
| "p99_kld": 0.13515481457212059, |
| "top1_agreement": 0.9672807602534178 |
| }, |
| "forced_assistant_prefix": { |
| "n": 68, |
| "mean_kld": 1.3652414722114897e-06, |
| "p99_kld": 3.1008135628809086e-05, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 680, |
| "mean_kld": 0.002393608337400068, |
| "p99_kld": 0.026955904789476196, |
| "top1_agreement": 1.0 |
| }, |
| "template_boundary": { |
| "n": 218, |
| "mean_kld": 0.011582153497749523, |
| "p99_kld": 0.15028413118666578, |
| "top1_agreement": 0.981651376146789 |
| }, |
| "user_prompt": { |
| "n": 6821, |
| "mean_kld": 0.02083493805611068, |
| "p99_kld": 0.2550854342190833, |
| "top1_agreement": 0.947075208913649 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 229, |
| "full_mean": 0.00761304665186533, |
| "approx_mean": 0.007612604811660384, |
| "mean_underestimate": 4.4184020494577824e-07, |
| "p99_abs_error": 9.42441787840273e-06, |
| "max_abs_error": 6.18071677417481e-05 |
| }, |
| "full_validation_prompt": { |
| "n": 43, |
| "full_mean": 0.002086134395148279, |
| "approx_mean": 0.001979007647009638, |
| "mean_underestimate": 0.0001071267481386407, |
| "p99_abs_error": 0.0002851239576901984, |
| "max_abs_error": 0.00032759181097957377 |
| }, |
| "all_position_decision_mean_kld": 0.006525804381694149, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 63, |
| "mean_kld": 0.009906346770346911, |
| "p99_kld": 0.09357772054828484, |
| "top1_agreement": 0.9682539682539683 |
| }, |
| "eos": { |
| "n": 63, |
| "mean_kld": 0.003235373734688004, |
| "p99_kld": 0.02061938202160945, |
| "top1_agreement": 1.0 |
| }, |
| "reverification_start": { |
| "n": 8, |
| "mean_kld": 0.005704580445518083, |
| "p99_kld": 0.016305374695063105, |
| "top1_agreement": 1.0 |
| }, |
| "think_close": { |
| "n": 63, |
| "mean_kld": 0.006602458757871321, |
| "p99_kld": 0.03568555945616072, |
| "top1_agreement": 0.9682539682539683 |
| }, |
| "tool_call_close": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| } |
| } |
| }, |
| "chinese": { |
| "n": 77925, |
| "mean_kld": 0.01018777641623211, |
| "p99_kld": 0.14902629800666964, |
| "top1_agreement": 0.9722297080526147, |
| "decision_n": 128, |
| "decision_mean_kld": 0.011815612402897555, |
| "decision_p99_kld": 0.13058346045550864, |
| "decision_top1_agreement": 0.9375, |
| "full_validation": { |
| "n": 168, |
| "full_mean": 0.00647821045937385, |
| "approx_mean": 0.006463348331810638, |
| "mean_underestimate": 1.486212756321256e-05, |
| "p99_abs_error": 0.00011119269574356159, |
| "max_abs_error": 0.00011119269574356159 |
| }, |
| "all_position_mean_kld": 0.01047254982264155, |
| "all_position_p99_kld": 0.15062938274033888, |
| "all_position_top1_agreement": 0.9720177096806288, |
| "assistant_n": 77925, |
| "all_position_n": 80408, |
| "prompt_n": 2483, |
| "prompt_mean_kld": 0.01940970958682023, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 44944, |
| "mean_kld": 0.01032318931627108, |
| "p99_kld": 0.15754258489019882, |
| "top1_agreement": 0.972187611249555 |
| }, |
| "assistant_thinking": { |
| "n": 32981, |
| "mean_kld": 0.010003246008441215, |
| "p99_kld": 0.13670941371102693, |
| "top1_agreement": 0.9722870743761559 |
| }, |
| "forced_assistant_prefix": { |
| "n": 46, |
| "mean_kld": 0.0012054051500934673, |
| "p99_kld": 0.030054882025588817, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 420, |
| "mean_kld": 0.002393608337400068, |
| "p99_kld": 0.026955904789476196, |
| "top1_agreement": 1.0 |
| }, |
| "template_boundary": { |
| "n": 151, |
| "mean_kld": 0.008237270657534565, |
| "p99_kld": 0.14212167880469936, |
| "top1_agreement": 0.9867549668874173 |
| }, |
| "user_prompt": { |
| "n": 1866, |
| "mean_kld": 0.024592559965795596, |
| "p99_kld": 0.2929908349881822, |
| "top1_agreement": 0.954983922829582 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 146, |
| "full_mean": 0.007435504804053342, |
| "approx_mean": 0.007434405088880526, |
| "mean_underestimate": 1.0997151728179588e-06, |
| "p99_abs_error": 2.886191019943536e-05, |
| "max_abs_error": 3.900864076853382e-05 |
| }, |
| "full_validation_prompt": { |
| "n": 22, |
| "full_mean": 0.0001252570810463023, |
| "approx_mean": 1.906258034683472e-05, |
| "mean_underestimate": 0.00010619450069946761, |
| "p99_abs_error": 0.00011119269574356159, |
| "max_abs_error": 0.00011119269574356159 |
| }, |
| "all_position_decision_mean_kld": 0.009397807892961269, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 43, |
| "mean_kld": 0.007035061474236651, |
| "p99_kld": 0.05686843091374827, |
| "top1_agreement": 0.9534883720930233 |
| }, |
| "eos": { |
| "n": 43, |
| "mean_kld": 0.006757991230333298, |
| "p99_kld": 0.07388981010579723, |
| "top1_agreement": 0.9534883720930233 |
| }, |
| "reverification_start": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| }, |
| "think_close": { |
| "n": 42, |
| "mean_kld": 0.02188802669700903, |
| "p99_kld": 0.14518641009670916, |
| "top1_agreement": 0.9047619047619048 |
| }, |
| "tool_call_close": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| } |
| } |
| }, |
| "code": { |
| "n": 137335, |
| "mean_kld": 0.0053867626699133425, |
| "p99_kld": 0.08701683985988576, |
| "top1_agreement": 0.982648268831689, |
| "decision_n": 146, |
| "decision_mean_kld": 0.0032024893321029946, |
| "decision_p99_kld": 0.03266172313634937, |
| "decision_top1_agreement": 0.9794520547945206, |
| "full_validation": { |
| "n": 376, |
| "full_mean": 0.004532167322740648, |
| "approx_mean": 0.004517339011632282, |
| "mean_underestimate": 1.4828311108365755e-05, |
| "p99_abs_error": 0.0001113973374174249, |
| "max_abs_error": 0.00035697392035272113 |
| }, |
| "all_position_mean_kld": 0.00796235823231685, |
| "all_position_p99_kld": 0.10666516424563804, |
| "all_position_top1_agreement": 0.9809259939246253, |
| "assistant_n": 137335, |
| "all_position_n": 149785, |
| "prompt_n": 12450, |
| "prompt_mean_kld": 0.03637355634980168, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 31245, |
| "mean_kld": 0.003155394478530517, |
| "p99_kld": 0.06870694125032602, |
| "top1_agreement": 0.9921907505200832 |
| }, |
| "assistant_thinking": { |
| "n": 106090, |
| "mean_kld": 0.006043932046289592, |
| "p99_kld": 0.0904233307758257, |
| "top1_agreement": 0.9798378735036289 |
| }, |
| "forced_assistant_prefix": { |
| "n": 106, |
| "mean_kld": 0.0009230918047764466, |
| "p99_kld": 0.02641106396448632, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 940, |
| "mean_kld": 0.0024339850489358155, |
| "p99_kld": 0.026955904789476196, |
| "top1_agreement": 1.0 |
| }, |
| "template_boundary": { |
| "n": 145, |
| "mean_kld": 0.03614585394040902, |
| "p99_kld": 0.5363758415261018, |
| "top1_agreement": 0.9724137931034482 |
| }, |
| "user_prompt": { |
| "n": 11259, |
| "mean_kld": 0.03954381686263129, |
| "p99_kld": 0.7856741219061122, |
| "top1_agreement": 0.9582556177280398 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 312, |
| "full_mean": 0.0050028832384544425, |
| "approx_mean": 0.005002956543808977, |
| "mean_underestimate": -7.330535453601083e-08, |
| "p99_abs_error": 1.8910166539799404e-06, |
| "max_abs_error": 2.8269323018501765e-06 |
| }, |
| "full_validation_prompt": { |
| "n": 64, |
| "full_mean": 0.002237427233635904, |
| "approx_mean": 0.002149953542270892, |
| "mean_underestimate": 8.747369136501187e-05, |
| "p99_abs_error": 0.00020420752373776917, |
| "max_abs_error": 0.00035697392035272113 |
| }, |
| "all_position_decision_mean_kld": 0.00306554878869674, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 48, |
| "mean_kld": 0.004129500844889752, |
| "p99_kld": 0.041779684875650175, |
| "top1_agreement": 1.0 |
| }, |
| "eos": { |
| "n": 48, |
| "mean_kld": 0.0023480717319386397, |
| "p99_kld": 0.022975867740248016, |
| "top1_agreement": 0.9791666666666666 |
| }, |
| "reverification_start": { |
| "n": 5, |
| "mean_kld": 0.008872724649792756, |
| "p99_kld": 0.029462845561513035, |
| "top1_agreement": 0.8 |
| }, |
| "think_close": { |
| "n": 45, |
| "mean_kld": 0.0024950296788957896, |
| "p99_kld": 0.02606781123677461, |
| "top1_agreement": 0.9777777777777777 |
| }, |
| "tool_call_close": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| } |
| } |
| }, |
| "longctx": { |
| "n": 5913, |
| "mean_kld": 0.0036857950088148186, |
| "p99_kld": 0.06011494283876536, |
| "top1_agreement": 0.9857940131912735, |
| "decision_n": 11, |
| "decision_mean_kld": 0.0022145571912580092, |
| "decision_p99_kld": 0.00935720736779331, |
| "decision_top1_agreement": 1.0, |
| "full_validation": { |
| "n": 24, |
| "full_mean": 0.16122826295644296, |
| "approx_mean": 0.16104310940643543, |
| "mean_underestimate": 0.00018515355000750226, |
| "p99_abs_error": 0.0021087679056120104, |
| "max_abs_error": 0.00260097256854748 |
| }, |
| "all_position_mean_kld": 0.06951449190438998, |
| "all_position_p99_kld": 1.620607876123471, |
| "all_position_top1_agreement": 0.9368849677638276, |
| "assistant_n": 5913, |
| "all_position_n": 53046, |
| "prompt_n": 47133, |
| "prompt_mean_kld": 0.07777293258806248, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 816, |
| "mean_kld": 0.0001883368305562633, |
| "p99_kld": 0.0059870625161263106, |
| "top1_agreement": 0.9987745098039216 |
| }, |
| "assistant_thinking": { |
| "n": 5097, |
| "mean_kld": 0.004245717683615482, |
| "p99_kld": 0.06561849957582837, |
| "top1_agreement": 0.9837159113203845 |
| }, |
| "forced_assistant_prefix": { |
| "n": 6, |
| "mean_kld": 3.5951277893944004e-11, |
| "p99_kld": 2.0492228399548088e-10, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 60, |
| "mean_kld": 0.0023936083374000686, |
| "p99_kld": 0.026955904789476196, |
| "top1_agreement": 1.0 |
| }, |
| "template_boundary": { |
| "n": 9, |
| "mean_kld": 0.0009587976298563324, |
| "p99_kld": 0.0043020575635540695, |
| "top1_agreement": 1.0 |
| }, |
| "user_prompt": { |
| "n": 47058, |
| "mean_kld": 0.07789365009124952, |
| "p99_kld": 1.8284055943509936, |
| "top1_agreement": 0.9306387861787582 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 4, |
| "full_mean": 0.00014196843021938417, |
| "approx_mean": 0.00014202123734670215, |
| "mean_underestimate": -5.280712731797232e-08, |
| "p99_abs_error": 2.6772403121952916e-07, |
| "max_abs_error": 2.7331743931999194e-07 |
| }, |
| "full_validation_prompt": { |
| "n": 20, |
| "full_mean": 0.19344552186168767, |
| "approx_mean": 0.1932233270402532, |
| "mean_underestimate": 0.00022219482143446629, |
| "p99_abs_error": 0.0021943687165573073, |
| "max_abs_error": 0.00260097256854748 |
| }, |
| "all_position_decision_mean_kld": 0.0023020026778306254, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 3, |
| "mean_kld": 0.0036607499845994255, |
| "p99_kld": 0.009428393142899175, |
| "top1_agreement": 1.0 |
| }, |
| "eos": { |
| "n": 3, |
| "mean_kld": 0.00018761861729277906, |
| "p99_kld": 0.000543785836606622, |
| "top1_agreement": 1.0 |
| }, |
| "reverification_start": { |
| "n": 2, |
| "mean_kld": 0.0064074348483322725, |
| "p99_kld": 0.007223407292001795, |
| "top1_agreement": 1.0 |
| }, |
| "think_close": { |
| "n": 3, |
| "mean_kld": 5.1200498981097837e-08, |
| "p99_kld": 1.393434327695629e-07, |
| "top1_agreement": 1.0 |
| }, |
| "tool_call_close": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| } |
| } |
| }, |
| "math": { |
| "n": 115237, |
| "mean_kld": 0.005996101616687342, |
| "p99_kld": 0.10857511637564023, |
| "top1_agreement": 0.9827225630656821, |
| "decision_n": 307, |
| "decision_mean_kld": 0.0034273899271458712, |
| "decision_p99_kld": 0.034344682742442005, |
| "decision_top1_agreement": 0.996742671009772, |
| "full_validation": { |
| "n": 648, |
| "full_mean": 0.004450932927728706, |
| "approx_mean": 0.004420456407656662, |
| "mean_underestimate": 3.047652007204392e-05, |
| "p99_abs_error": 0.00048254821396412145, |
| "max_abs_error": 0.00359806857436884 |
| }, |
| "all_position_mean_kld": 0.006673647803463926, |
| "all_position_p99_kld": 0.1136013718764783, |
| "all_position_top1_agreement": 0.9818860056361889, |
| "assistant_n": 115237, |
| "all_position_n": 122778, |
| "prompt_n": 7541, |
| "prompt_mean_kld": 0.017027498741744404, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 22608, |
| "mean_kld": 0.0023123110881905725, |
| "p99_kld": 0.04295761645535131, |
| "top1_agreement": 0.9932767162066525 |
| }, |
| "assistant_thinking": { |
| "n": 92629, |
| "mean_kld": 0.006895205960556487, |
| "p99_kld": 0.11800064236350856, |
| "top1_agreement": 0.9801466063543814 |
| }, |
| "forced_assistant_prefix": { |
| "n": 202, |
| "mean_kld": 0.0010899746370388308, |
| "p99_kld": 0.03744179784199983, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 1620, |
| "mean_kld": 0.002892672712437554, |
| "p99_kld": 0.026955904789476196, |
| "top1_agreement": 1.0 |
| }, |
| "template_boundary": { |
| "n": 243, |
| "mean_kld": 0.033217920366317065, |
| "p99_kld": 0.571716909950343, |
| "top1_agreement": 0.9670781893004116 |
| }, |
| "user_prompt": { |
| "n": 5476, |
| "mean_kld": 0.02107854431914697, |
| "p99_kld": 0.2647954115323116, |
| "top1_agreement": 0.9589116143170198 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 495, |
| "full_mean": 0.0030770658464762666, |
| "approx_mean": 0.003077100001016382, |
| "mean_underestimate": -3.415454011622988e-08, |
| "p99_abs_error": 3.0228543338483252e-06, |
| "max_abs_error": 4.552360838868937e-06 |
| }, |
| "full_validation_prompt": { |
| "n": 153, |
| "full_mean": 0.00889579701413366, |
| "approx_mean": 0.00876660948796345, |
| "mean_underestimate": 0.0001291875261702091, |
| "p99_abs_error": 0.0014083899713788523, |
| "max_abs_error": 0.00359806857436884 |
| }, |
| "all_position_decision_mean_kld": 0.003913345745636584, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 81, |
| "mean_kld": 0.004451630096904059, |
| "p99_kld": 0.03924023245324257, |
| "top1_agreement": 1.0 |
| }, |
| "eos": { |
| "n": 81, |
| "mean_kld": 0.0006871849125211745, |
| "p99_kld": 0.01116196726629134, |
| "top1_agreement": 1.0 |
| }, |
| "reverification_start": { |
| "n": 74, |
| "mean_kld": 0.008202922054398083, |
| "p99_kld": 0.043869944560617324, |
| "top1_agreement": 0.9864864864864865 |
| }, |
| "think_close": { |
| "n": 71, |
| "mean_kld": 0.00040772478654761284, |
| "p99_kld": 0.007182759127872344, |
| "top1_agreement": 1.0 |
| }, |
| "tool_call_close": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| } |
| } |
| }, |
| "multilingual": { |
| "n": 47621, |
| "mean_kld": 0.009154899380522889, |
| "p99_kld": 0.15035925043558743, |
| "top1_agreement": 0.9765649608366057, |
| "decision_n": 30, |
| "decision_mean_kld": 0.006640274059774302, |
| "decision_p99_kld": 0.0815537881622804, |
| "decision_top1_agreement": 1.0, |
| "full_validation": { |
| "n": 72, |
| "full_mean": 0.003718575936033885, |
| "approx_mean": 0.003702089055298991, |
| "mean_underestimate": 1.6486880734893288e-05, |
| "p99_abs_error": 0.00011119269574356159, |
| "max_abs_error": 0.00011119269574356159 |
| }, |
| "all_position_mean_kld": 0.009641340550365744, |
| "all_position_p99_kld": 0.15183046787338655, |
| "all_position_top1_agreement": 0.9750260145681582, |
| "assistant_n": 47621, |
| "all_position_n": 49972, |
| "prompt_n": 2351, |
| "prompt_mean_kld": 0.019494515773286467, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 12081, |
| "mean_kld": 0.009247482440945129, |
| "p99_kld": 0.1573713572769374, |
| "top1_agreement": 0.9761609138316365 |
| }, |
| "assistant_thinking": { |
| "n": 35540, |
| "mean_kld": 0.00912342791307885, |
| "p99_kld": 0.1468408389974018, |
| "top1_agreement": 0.976702307259426 |
| }, |
| "forced_assistant_prefix": { |
| "n": 22, |
| "mean_kld": 0.0037881282589971267, |
| "p99_kld": 0.05670937767580077, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 180, |
| "mean_kld": 0.0023936083374000677, |
| "p99_kld": 0.026955904789476196, |
| "top1_agreement": 1.0 |
| }, |
| "template_boundary": { |
| "n": 27, |
| "mean_kld": 0.03232473998679162, |
| "p99_kld": 0.5978049262937608, |
| "top1_agreement": 0.9629629629629629 |
| }, |
| "user_prompt": { |
| "n": 2122, |
| "mean_kld": 0.020944698530124012, |
| "p99_kld": 0.1681770136305531, |
| "top1_agreement": 0.9382657869934025 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 62, |
| "full_mean": 0.004271837832924583, |
| "approx_mean": 0.00426945492025055, |
| "mean_underestimate": 2.38291267403328e-06, |
| "p99_abs_error": 6.238807188824175e-05, |
| "max_abs_error": 0.00010016771249762076 |
| }, |
| "full_validation_prompt": { |
| "n": 10, |
| "full_mean": 0.00028835217531155067, |
| "approx_mean": 0.00018442069259932538, |
| "mean_underestimate": 0.00010393148271222534, |
| "p99_abs_error": 0.00011119269574356159, |
| "max_abs_error": 0.00011119269574356159 |
| }, |
| "all_position_decision_mean_kld": 0.014298554832657007, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 9, |
| "mean_kld": 0.013583020410426775, |
| "p99_kld": 0.09387241620196148, |
| "top1_agreement": 1.0 |
| }, |
| "eos": { |
| "n": 9, |
| "mean_kld": 0.004748232265103456, |
| "p99_kld": 0.03128454372070919, |
| "top1_agreement": 1.0 |
| }, |
| "reverification_start": { |
| "n": 4, |
| "mean_kld": 0.008173042569305265, |
| "p99_kld": 0.013180960943480424, |
| "top1_agreement": 1.0 |
| }, |
| "think_close": { |
| "n": 8, |
| "mean_kld": 0.00019184717952948916, |
| "p99_kld": 0.0014179659709771337, |
| "top1_agreement": 1.0 |
| }, |
| "tool_call_close": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| } |
| } |
| }, |
| "stem": { |
| "n": 29823, |
| "mean_kld": 0.011514940530525668, |
| "p99_kld": 0.15804505122360546, |
| "top1_agreement": 0.9703584481775811, |
| "decision_n": 13, |
| "decision_mean_kld": 0.0007897160983312732, |
| "decision_p99_kld": 0.004770724279739736, |
| "decision_top1_agreement": 1.0, |
| "full_validation": { |
| "n": 32, |
| "full_mean": 0.010714772222544456, |
| "approx_mean": 0.010700660320573478, |
| "mean_underestimate": 1.4111901970977644e-05, |
| "p99_abs_error": 0.00011119269574356159, |
| "max_abs_error": 0.00011119269574356159 |
| }, |
| "all_position_mean_kld": 0.011657505294592127, |
| "all_position_p99_kld": 0.1593496302859785, |
| "all_position_top1_agreement": 0.969888231426693, |
| "assistant_n": 29823, |
| "all_position_n": 30420, |
| "prompt_n": 597, |
| "prompt_mean_kld": 0.018779295845268867, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 13964, |
| "mean_kld": 0.009962160290646314, |
| "p99_kld": 0.16003601200209416, |
| "top1_agreement": 0.9778000572901747 |
| }, |
| "assistant_thinking": { |
| "n": 15859, |
| "mean_kld": 0.012882178267436904, |
| "p99_kld": 0.1570943418385551, |
| "top1_agreement": 0.9638060407339681 |
| }, |
| "forced_assistant_prefix": { |
| "n": 8, |
| "mean_kld": 2.579764911270815e-09, |
| "p99_kld": 1.8997070744224476e-08, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 80, |
| "mean_kld": 0.002393608337400068, |
| "p99_kld": 0.026955904789476196, |
| "top1_agreement": 1.0 |
| }, |
| "template_boundary": { |
| "n": 12, |
| "mean_kld": 0.011376777095675817, |
| "p99_kld": 0.1101403158120921, |
| "top1_agreement": 1.0 |
| }, |
| "user_prompt": { |
| "n": 497, |
| "mean_kld": 0.021897846291443215, |
| "p99_kld": 0.3266325668064325, |
| "top1_agreement": 0.9356136820925554 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 28, |
| "full_mean": 0.012229167174524535, |
| "approx_mean": 0.01222892395737821, |
| "mean_underestimate": 2.432171463227957e-07, |
| "p99_abs_error": 5.3237603356941346e-06, |
| "max_abs_error": 7.130772233729377e-06 |
| }, |
| "full_validation_prompt": { |
| "n": 4, |
| "full_mean": 0.00011400755868390622, |
| "approx_mean": 2.814862940344633e-06, |
| "mean_underestimate": 0.00011119269574356159, |
| "p99_abs_error": 0.00011119269574356159, |
| "max_abs_error": 0.00011119269574356159 |
| }, |
| "all_position_decision_mean_kld": 0.0012173652040185273, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 4, |
| "mean_kld": 0.0019222251740729848, |
| "p99_kld": 0.004996275592724057, |
| "top1_agreement": 1.0 |
| }, |
| "eos": { |
| "n": 4, |
| "mean_kld": 2.320010651984801e-06, |
| "p99_kld": 4.4747945586141115e-06, |
| "top1_agreement": 1.0 |
| }, |
| "reverification_start": { |
| "n": 1, |
| "mean_kld": 0.0024767621409890286, |
| "p99_kld": 0.0024767621409890286, |
| "top1_agreement": 1.0 |
| }, |
| "think_close": { |
| "n": 4, |
| "mean_kld": 2.28415996044115e-05, |
| "p99_kld": 8.85744564773058e-05, |
| "top1_agreement": 1.0 |
| }, |
| "tool_call_close": { |
| "n": 0, |
| "mean_kld": null, |
| "p99_kld": null, |
| "top1_agreement": null |
| } |
| } |
| }, |
| "structured": { |
| "n": 37742, |
| "mean_kld": 0.004372234329206234, |
| "p99_kld": 0.07347531507253739, |
| "top1_agreement": 0.985347888294208, |
| "decision_n": 91, |
| "decision_mean_kld": 0.0017239214791775926, |
| "decision_p99_kld": 0.03706113130504692, |
| "decision_top1_agreement": 0.978021978021978, |
| "full_validation": { |
| "n": 128, |
| "full_mean": 0.003729116494500936, |
| "approx_mean": 0.003706420610225084, |
| "mean_underestimate": 2.2695884275851733e-05, |
| "p99_abs_error": 0.00013847499421678242, |
| "max_abs_error": 0.0006768156143293694 |
| }, |
| "all_position_mean_kld": 0.014707125886740132, |
| "all_position_p99_kld": 0.19275899980820355, |
| "all_position_top1_agreement": 0.9778924834443711, |
| "assistant_n": 37742, |
| "all_position_n": 49983, |
| "prompt_n": 12241, |
| "prompt_mean_kld": 0.0465721268804861, |
| "headline_segment_rule": "assistant content: thinking, answer, tool call; includes historical assistant content; excludes fixed headers and injected thinking prefix", |
| "by_segment": { |
| "assistant_answer": { |
| "n": 8607, |
| "mean_kld": 0.004387050213706964, |
| "p99_kld": 0.08446986111363049, |
| "top1_agreement": 0.985709306378529 |
| }, |
| "assistant_thinking": { |
| "n": 26110, |
| "mean_kld": 0.004868900199142164, |
| "p99_kld": 0.07856574528500437, |
| "top1_agreement": 0.9835312140942167 |
| }, |
| "assistant_tool_call": { |
| "n": 3025, |
| "mean_kld": 4.3154599644269495e-05, |
| "p99_kld": 3.6668231357857075e-05, |
| "top1_agreement": 1.0 |
| }, |
| "forced_assistant_prefix": { |
| "n": 36, |
| "mean_kld": 0.00032010079352116477, |
| "p99_kld": 0.007387549646212401, |
| "top1_agreement": 1.0 |
| }, |
| "system/tool-schema": { |
| "n": 7905, |
| "mean_kld": 0.0610653970552492, |
| "p99_kld": 1.178408437880272, |
| "top1_agreement": 0.948134092346616 |
| }, |
| "template_boundary": { |
| "n": 52, |
| "mean_kld": 0.0033323353850341127, |
| "p99_kld": 0.037740997689761864, |
| "top1_agreement": 1.0 |
| }, |
| "user_prompt": { |
| "n": 4248, |
| "mean_kld": 0.020523219480625444, |
| "p99_kld": 0.2716690768829108, |
| "top1_agreement": 0.966572504708098 |
| } |
| }, |
| "full_validation_assistant": { |
| "n": 81, |
| "full_mean": 0.003229712943395707, |
| "approx_mean": 0.0032296496850528064, |
| "mean_underestimate": 6.325834290010256e-08, |
| "p99_abs_error": 2.065936184858916e-06, |
| "max_abs_error": 6.774203776704901e-06 |
| }, |
| "full_validation_prompt": { |
| "n": 47, |
| "full_mean": 0.00458979069959718, |
| "approx_mean": 0.0045280896514794345, |
| "mean_underestimate": 6.170104811774708e-05, |
| "p99_abs_error": 0.0004334742006605297, |
| "max_abs_error": 0.0006768156143293694 |
| }, |
| "all_position_decision_mean_kld": 0.005209399166786532, |
| "decision_by_kind": { |
| "answer_start": { |
| "n": 17, |
| "mean_kld": 0.0009144063671686316, |
| "p99_kld": 0.01230856427285549, |
| "top1_agreement": 0.9411764705882353 |
| }, |
| "eos": { |
| "n": 17, |
| "mean_kld": 0.0010946281687069953, |
| "p99_kld": 0.011514810199731253, |
| "top1_agreement": 1.0 |
| }, |
| "reverification_start": { |
| "n": 12, |
| "mean_kld": 0.008977896341437858, |
| "p99_kld": 0.05149705612124191, |
| "top1_agreement": 0.9166666666666666 |
| }, |
| "think_close": { |
| "n": 16, |
| "mean_kld": 0.000936781962371425, |
| "p99_kld": 0.007793534069028571, |
| "top1_agreement": 1.0 |
| }, |
| "tool_call_close": { |
| "n": 29, |
| "mean_kld": 2.6955812939555743e-15, |
| "p99_kld": 5.4768775482676576e-14, |
| "top1_agreement": 1.0 |
| } |
| } |
| } |
| }, |
| "metrics_revision": "assistant-only full-vocab validation; CPU-only cached arrays", |
| "execution_mode": "teacher-forced EXTEND/prefill", |
| "kv_cache_dtype": "auto" |
| }, |
| "native20_decode_referee": { |
| "corpus": "native-control-20, artifact-generated continuations; not frozen eval v1", |
| "execution_mode": "uninstrumented native autoregressive DECODE vs BF16 native prefill referee", |
| "kv_cache_dtype": "auto (verified physical BF16)", |
| "estimator": "intersection of native decode/prefill top20, plus aggregate complement", |
| "direction": "KL(BF16 prefill || NVFP4/FP8 native path), natural logs", |
| "exclude_first_generated_token": true, |
| "overall": { |
| "n_tokens": 4711, |
| "n_prompts": 20, |
| "native_decode_to_prefill_kl": 0.0036020699905831375, |
| "native_decode_prefill_top1_flip": 0.014858841010401188, |
| "bf16_to_decode_kl": 0.0043261702937709015, |
| "bf16_to_prefill_kl": 0.004186384126407461, |
| "decode_minus_prefill_kl": 0.00013978616736344047, |
| "bf16_decode_top1_flip": 0.017830609212481426, |
| "bf16_prefill_top1_flip": 0.015495648482275526, |
| "shared_support_size": 19.198896200382084, |
| "bf16_tail_mass": 0.0015594668676902537, |
| "decode_worse_fraction": 0.48864360008490765 |
| }, |
| "by_domain": { |
| "agentic": { |
| "n_tokens": 886, |
| "n_prompts": 5, |
| "native_decode_to_prefill_kl": 0.001386351934703459, |
| "native_decode_prefill_top1_flip": 0.002257336343115124, |
| "bf16_to_decode_kl": 0.0015817936866103998, |
| "bf16_to_prefill_kl": 0.0018863917310804775, |
| "decode_minus_prefill_kl": -0.00030459804447007774, |
| "bf16_decode_top1_flip": 0.003386004514672686, |
| "bf16_prefill_top1_flip": 0.0056433408577878106, |
| "shared_support_size": 19.2313769751693, |
| "bf16_tail_mass": 2.156054715545636e-05, |
| "decode_worse_fraction": 0.44469525959367945 |
| }, |
| "math": { |
| "n_tokens": 1275, |
| "n_prompts": 5, |
| "native_decode_to_prefill_kl": 0.0020061534647821217, |
| "native_decode_prefill_top1_flip": 0.013333333333333334, |
| "bf16_to_decode_kl": 0.0024727424437004457, |
| "bf16_to_prefill_kl": 0.0021211202068311618, |
| "decode_minus_prefill_kl": 0.00035162223686928406, |
| "bf16_decode_top1_flip": 0.012549019607843137, |
| "bf16_prefill_top1_flip": 0.007058823529411765, |
| "shared_support_size": 19.298823529411763, |
| "bf16_tail_mass": 0.001596943575401765, |
| "decode_worse_fraction": 0.508235294117647 |
| }, |
| "code": { |
| "n_tokens": 1275, |
| "n_prompts": 5, |
| "native_decode_to_prefill_kl": 0.0021951403766355612, |
| "native_decode_prefill_top1_flip": 0.01019607843137255, |
| "bf16_to_decode_kl": 0.0025578031282717634, |
| "bf16_to_prefill_kl": 0.00266926028214078, |
| "decode_minus_prefill_kl": -0.00011145715386901665, |
| "bf16_decode_top1_flip": 0.01411764705882353, |
| "bf16_prefill_top1_flip": 0.014901960784313726, |
| "shared_support_size": 19.173333333333332, |
| "bf16_tail_mass": 0.0005147679044791591, |
| "decode_worse_fraction": 0.47294117647058825 |
| }, |
| "chinese": { |
| "n_tokens": 1275, |
| "n_prompts": 5, |
| "native_decode_to_prefill_kl": 0.008144622951907725, |
| "native_decode_prefill_top1_flip": 0.02980392156862745, |
| "bf16_to_decode_kl": 0.00985503799478693, |
| "bf16_to_prefill_kl": 0.009367041115552173, |
| "decode_minus_prefill_kl": 0.0004879968792347575, |
| "bf16_decode_top1_flip": 0.03686274509803922, |
| "bf16_prefill_top1_flip": 0.03137254901960784, |
| "shared_support_size": 19.101960784313725, |
| "bf16_tail_mass": 0.0036353832408320564, |
| "decode_worse_fraction": 0.5152941176470588 |
| } |
| }, |
| "source": "quant-fp8-nvfp4/results/native-v1/fp8-block-calib-v1-auto.json", |
| "source_sha256": "3f52c30ade938ed8a73c644a2781e1e95b8b1d16a3c8f302e54b699656c95482" |
| }, |
| "exact_stop_bias": { |
| "artifact": "fp8-block-calib-v1", |
| "reference": "bf16", |
| "execution": { |
| "mode": "teacher-forced native EXTEND/prefill", |
| "runtime": "SGLang 0.5.21", |
| "kv_cache_dtype": "auto" |
| }, |
| "reference_execution": { |
| "mode": "teacher-forced native EXTEND/prefill", |
| "runtime": "SGLang 0.5.21", |
| "kv_cache_dtype": "bfloat16" |
| }, |
| "estimator": "exact focused full-vocabulary log-softmax probabilities; no top-k approximation", |
| "ci": "95% conversation-cluster bootstrap,4000 fixed-seed draws; token positions are not independent samples", |
| "scope": "current native BF16 response only; assistant history excluded", |
| "reverification_proxy": "Sum exact mass of five distinct observed first-token IDs (Wait/wait/Let variants) at thinking positions. This is a lexical start proxy, not the joint probability of a multi-token phrase or a semantic loop probability.", |
| "domains": { |
| "all": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 246, |
| "n_conversations": 246, |
| "mean": -0.0017175679949844908, |
| "ci95": [ |
| -0.006240826371856821, |
| 0.002778891956343598 |
| ] |
| }, |
| "eos": { |
| "n_positions": 272, |
| "n_conversations": 272, |
| "mean": 0.004252021895589408, |
| "ci95": [ |
| -0.0012373934207318157, |
| 0.011246997364874797 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 241, |
| "n_conversations": 241, |
| "mean": -0.008197844929359627, |
| "ci95": [ |
| -0.01836565367568048, |
| 0.0023626459839510795 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 122, |
| "n_conversations": 47, |
| "mean": -0.006190959354162968, |
| "ci95": [ |
| -0.02455500616110225, |
| 0.016807605755787155 |
| ] |
| }, |
| "tool_call_close": { |
| "n_positions": 119, |
| "n_conversations": 69, |
| "mean": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 395963, |
| "n_conversations": 252, |
| "mean": 7.772027464666808e-08, |
| "ci95": [ |
| -2.559651440957416e-06, |
| 2.7626495583434473e-06 |
| ], |
| "quant_mass_sum": 250.490322855317, |
| "bf16_mass_sum": 250.4595485022071, |
| "quant_mean_mass": 0.0006326104278816884, |
| "bf16_mean_mass": 0.0006325327076070419, |
| "mass_ratio": 1.0001228715506914, |
| "mass_ratio_ci95": [ |
| 0.9961347635000831, |
| 1.0043195154872775 |
| ], |
| "constant_hazard_predicted_length_ratio": 0.9998771435448719, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 180905, |
| "n_conversations": 266, |
| "mean": 5.672458978802684e-06, |
| "ci95": [ |
| 2.920022870768618e-07, |
| 1.140297229442154e-05 |
| ], |
| "quant_mass_sum": 270.50416259032113, |
| "bf16_mass_sum": 269.4779863987609, |
| "quant_mean_mass": 0.001495282952877594, |
| "bf16_mean_mass": 0.0014896104938987916, |
| "mass_ratio": 1.0038080149153326, |
| "mass_ratio_ci95": [ |
| 1.0002026943336149, |
| 1.0077773990173753 |
| ] |
| }, |
| "tool_close": { |
| "n_positions": 12923, |
| "n_conversations": 63, |
| "mean": -1.588150525834926e-08, |
| "ci95": [ |
| -6.095319197697469e-08, |
| 5.515922729952808e-10 |
| ], |
| "quant_mass_sum": 111.00091062344184, |
| "bf16_mass_sum": 111.00111586013428, |
| "quant_mean_mass": 0.008589407306619348, |
| "bf16_mean_mass": 0.008589423188124605, |
| "mass_ratio": 0.9999981510393761, |
| "mass_ratio_ci95": [ |
| 0.9999939599822164, |
| 1.0000000625575827 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 395963, |
| "n_conversations": 252, |
| "mean": 4.2482671090028906e-06, |
| "ci95": [ |
| -8.257781130705067e-06, |
| 1.785588115984088e-05 |
| ], |
| "quant_mass_sum": 1597.210980548646, |
| "bf16_mass_sum": 1595.5288239593635, |
| "quant_mean_mass": 0.004033737951648629, |
| "bf16_mean_mass": 0.004029489684539625, |
| "mass_ratio": 1.0010542940773128, |
| "mass_ratio_ci95": [ |
| 0.9980226822443511, |
| 1.0044695215435335 |
| ] |
| } |
| } |
| }, |
| "agentic": { |
| "decision_logp_bias": { |
| "eos": { |
| "n_positions": 57, |
| "n_conversations": 57, |
| "mean": -0.0015457907415622142, |
| "ci95": [ |
| -0.011085097890801926, |
| 0.00814070877042883 |
| ] |
| }, |
| "tool_call_close": { |
| "n_positions": 90, |
| "n_conversations": 57, |
| "mean": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 16, |
| "n_conversations": 8, |
| "mean": 0.04745988454669714, |
| "ci95": [ |
| -0.017753529184394413, |
| 0.18987967818975449 |
| ] |
| }, |
| "think_close": { |
| "n_positions": 47, |
| "n_conversations": 47, |
| "mean": 0.011890322142751254, |
| "ci95": [ |
| -0.001182286836642851, |
| 0.026088992727005808 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 26, |
| "n_conversations": 26, |
| "mean": 0.008252179679962305, |
| "ci95": [ |
| -0.029882782239180342, |
| 0.04619654821805086 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 33885, |
| "n_conversations": 53, |
| "mean": 1.2611872665188372e-05, |
| "ci95": [ |
| -7.917049597613983e-07, |
| 3.4543815794568344e-05 |
| ], |
| "quant_mass_sum": 48.863894401638596, |
| "bf16_mass_sum": 48.43654109637869, |
| "quant_mean_mass": 0.001442050889822594, |
| "bf16_mean_mass": 0.0014294390171574057, |
| "mass_ratio": 1.0088229525805643, |
| "mass_ratio_ci95": [ |
| 0.9994394485332742, |
| 1.0189569198156598 |
| ], |
| "constant_hazard_predicted_length_ratio": 0.9912542111001784, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 1270, |
| "n_conversations": 51, |
| "mean": -0.00024136024801527792, |
| "ci95": [ |
| -0.0005267105666998579, |
| -3.2749342256017385e-05 |
| ], |
| "quant_mass_sum": 52.647276773364716, |
| "bf16_mass_sum": 52.95380428834412, |
| "quant_mean_mass": 0.041454548640444656, |
| "bf16_mean_mass": 0.041695908888459934, |
| "mass_ratio": 0.9942114165526182, |
| "mass_ratio_ci95": [ |
| 0.9891327146332983, |
| 0.9991781432302328 |
| ] |
| }, |
| "tool_close": { |
| "n_positions": 9898, |
| "n_conversations": 51, |
| "mean": -2.07350813009799e-08, |
| "ci95": [ |
| -8.133297280140855e-08, |
| 7.886074272576047e-10 |
| ], |
| "quant_mass_sum": 82.0009098660758, |
| "bf16_mass_sum": 82.00111510191051, |
| "quant_mean_mass": 0.008284593843814489, |
| "bf16_mean_mass": 0.008284614578895788, |
| "mass_ratio": 0.9999974971580026, |
| "mass_ratio_ci95": [ |
| 0.9999916221284109, |
| 1.0000000931270074 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 33885, |
| "n_conversations": 53, |
| "mean": 0.00010943280188905057, |
| "ci95": [ |
| 6.082497114938725e-05, |
| 0.00017665045546841815 |
| ], |
| "quant_mass_sum": 211.22648788626094, |
| "bf16_mass_sum": 207.51835739425047, |
| "quant_mean_mass": 0.006233628091670679, |
| "bf16_mean_mass": 0.006124195289781628, |
| "mass_ratio": 1.017868927542471, |
| "mass_ratio_ci95": [ |
| 1.0102610373017975, |
| 1.0251961846032822 |
| ] |
| } |
| } |
| }, |
| "chat": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 34, |
| "n_conversations": 34, |
| "mean": -0.006124049592336517, |
| "ci95": [ |
| -0.022229253891074517, |
| 0.005045265056445842 |
| ] |
| }, |
| "eos": { |
| "n_positions": 34, |
| "n_conversations": 34, |
| "mean": 0.005205865115920674, |
| "ci95": [ |
| 0.000598427410005463, |
| 0.012025171769773003 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 34, |
| "n_conversations": 34, |
| "mean": -0.011004245811370631, |
| "ci95": [ |
| -0.05312379399500892, |
| 0.03656205675286032 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 8, |
| "n_conversations": 3, |
| "mean": 0.0073358339723341714, |
| "ci95": [ |
| -0.025093768723290093, |
| 0.20430755615234375 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 47868, |
| "n_conversations": 34, |
| "mean": -3.3226593469009573e-06, |
| "ci95": [ |
| -1.2964199222034087e-05, |
| 7.857401521161395e-06 |
| ], |
| "quant_mass_sum": 33.186666268786496, |
| "bf16_mass_sum": 33.34571532640395, |
| "quant_mean_mass": 0.0006932954430681561, |
| "bf16_mean_mass": 0.000696618102415057, |
| "mass_ratio": 0.9952303000232383, |
| "mass_ratio_ci95": [ |
| 0.9804128812683942, |
| 1.0105675656905482 |
| ], |
| "constant_hazard_predicted_length_ratio": 1.0047925590455298, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 54142, |
| "n_conversations": 34, |
| "mean": 7.09292698791355e-06, |
| "ci95": [ |
| 2.7040548516027407e-07, |
| 1.4562542824044871e-05 |
| ], |
| "quant_mass_sum": 34.8699979785914, |
| "bf16_mass_sum": 34.48597272561179, |
| "quant_mean_mass": 0.0006440470979755347, |
| "bf16_mean_mass": 0.0006369541709876212, |
| "mass_ratio": 1.0111356943889944, |
| "mass_ratio_ci95": [ |
| 1.0004270961799484, |
| 1.0211105089473547 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 47868, |
| "n_conversations": 34, |
| "mean": -7.534609658141529e-06, |
| "ci95": [ |
| -4.4349788931390064e-05, |
| 2.4253917304696784e-05 |
| ], |
| "quant_mass_sum": 154.71660634447113, |
| "bf16_mass_sum": 155.07727303958706, |
| "quant_mean_mass": 0.0032321510475572645, |
| "bf16_mean_mass": 0.003239685657215406, |
| "mass_ratio": 0.9976742775517864, |
| "mass_ratio_ci95": [ |
| 0.9869427564346173, |
| 1.0078813213755073 |
| ] |
| } |
| } |
| }, |
| "chinese": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 20, |
| "n_conversations": 20, |
| "mean": -0.019857615906346382, |
| "ci95": [ |
| -0.040011390837340455, |
| -0.003876992044390587 |
| ] |
| }, |
| "eos": { |
| "n_positions": 21, |
| "n_conversations": 21, |
| "mean": 0.027040740808350467, |
| "ci95": [ |
| -0.01793776404668663, |
| 0.09644747785866466 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 21, |
| "n_conversations": 21, |
| "mean": -0.012448191605341006, |
| "ci95": [ |
| -0.05794667304560003, |
| 0.025471730377916644 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 32893, |
| "n_conversations": 20, |
| "mean": -7.3509227373540665e-06, |
| "ci95": [ |
| -1.6960344478098604e-05, |
| -2.2242020474856465e-06 |
| ], |
| "quant_mass_sum": 17.782921902054166, |
| "bf16_mass_sum": 18.024715803653955, |
| "quant_mean_mass": 0.0005406293710532383, |
| "bf16_mean_mass": 0.0005479802937905923, |
| "mass_ratio": 0.9865854250223034, |
| "mass_ratio_ci95": [ |
| 0.9738737503935568, |
| 0.9958963458385534 |
| ], |
| "constant_hazard_predicted_length_ratio": 1.01359697258592, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 37427, |
| "n_conversations": 21, |
| "mean": 2.4963139115503667e-06, |
| "ci95": [ |
| -1.560282216122118e-05, |
| 2.024339332728116e-05 |
| ], |
| "quant_mass_sum": 22.457429156265288, |
| "bf16_mass_sum": 22.36399961549769, |
| "quant_mean_mass": 0.0006000328414317281, |
| "bf16_mean_mass": 0.0005975365275201777, |
| "mass_ratio": 1.0041776758349994, |
| "mass_ratio_ci95": [ |
| 0.9784152822332007, |
| 1.0388702300762658 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 32893, |
| "n_conversations": 20, |
| "mean": 7.477334229243242e-06, |
| "ci95": [ |
| -1.8718020586514283e-06, |
| 1.8705172336966093e-05 |
| ], |
| "quant_mass_sum": 39.457369650470596, |
| "bf16_mass_sum": 39.2114176956681, |
| "quant_mean_mass": 0.0011995673745316814, |
| "bf16_mean_mass": 0.0011920900403024383, |
| "mass_ratio": 1.006272457596698, |
| "mass_ratio_ci95": [ |
| 0.99815747299163, |
| 1.0257347832423251 |
| ] |
| } |
| } |
| }, |
| "code": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 44, |
| "n_conversations": 44, |
| "mean": 0.0019095236159602665, |
| "ci95": [ |
| -0.00619704802958242, |
| 0.009758830381577415 |
| ] |
| }, |
| "eos": { |
| "n_positions": 47, |
| "n_conversations": 47, |
| "mean": 0.0011943659852312958, |
| "ci95": [ |
| -0.014911737938881748, |
| 0.016804854875550442 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 47, |
| "n_conversations": 47, |
| "mean": -0.012199728603447627, |
| "ci95": [ |
| -0.025673506033031552, |
| 0.00017049748162111154 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 5, |
| "n_conversations": 4, |
| "mean": -0.03774479180574417, |
| "ci95": [ |
| -0.0769436702132225, |
| -0.004775635898113251 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 106086, |
| "n_conversations": 44, |
| "mean": 9.762680546469771e-07, |
| "ci95": [ |
| -3.171652438872643e-06, |
| 5.12124658519869e-06 |
| ], |
| "quant_mass_sum": 46.98783231888763, |
| "bf16_mass_sum": 46.884263946042346, |
| "quant_mean_mass": 0.0004429220850902817, |
| "bf16_mean_mass": 0.00044194581703563474, |
| "mass_ratio": 1.0022090220497963, |
| "mass_ratio_ci95": [ |
| 0.9934794878549621, |
| 1.0114906423299783 |
| ], |
| "constant_hazard_predicted_length_ratio": 0.9977958469728417, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 30741, |
| "n_conversations": 47, |
| "mean": 1.3598234743783117e-05, |
| "ci95": [ |
| -1.0329334139800924e-06, |
| 2.9669586569476473e-05 |
| ], |
| "quant_mass_sum": 46.56745830849335, |
| "bf16_mass_sum": 46.14943497423471, |
| "quant_mean_mass": 0.0015148322536187291, |
| "bf16_mean_mass": 0.0015012340188749458, |
| "mass_ratio": 1.0090580379693062, |
| "mass_ratio_ci95": [ |
| 0.9992620684416316, |
| 1.018629653011199 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 106086, |
| "n_conversations": 44, |
| "mean": -1.4943120076381688e-05, |
| "ci95": [ |
| -3.4408013862744794e-05, |
| 2.033139463641548e-06 |
| ], |
| "quant_mass_sum": 321.2247277374864, |
| "bf16_mass_sum": 322.80998357390945, |
| "quant_mean_mass": 0.0030279653086881055, |
| "bf16_mean_mass": 0.003042908428764488, |
| "mass_ratio": 0.9950891982370795, |
| "mass_ratio_ci95": [ |
| 0.9889591451014998, |
| 1.0007020220181289 |
| ] |
| } |
| } |
| }, |
| "longctx": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 3, |
| "n_conversations": 3, |
| "mean": -3.1230350335439044e-07, |
| "ci95": [ |
| -9.369105100631714e-07, |
| 0.0 |
| ] |
| }, |
| "eos": { |
| "n_positions": 3, |
| "n_conversations": 3, |
| "mean": 0.0006926460124911197, |
| "ci95": [ |
| -1.1734664440155029e-07, |
| 0.002059340011328459 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 3, |
| "n_conversations": 3, |
| "mean": -0.02360119259295364, |
| "ci95": [ |
| -0.06745445728302002, |
| 0.0 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 2, |
| "n_conversations": 1, |
| "mean": 0.08964859694242477, |
| "ci95": [ |
| 0.08964859694242477, |
| 0.08964859694242477 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 5097, |
| "n_conversations": 3, |
| "mean": 4.12567317040157e-06, |
| "ci95": [ |
| 1.1303711938994593e-09, |
| 1.6725600260163535e-05 |
| ], |
| "quant_mass_sum": 3.4742177994595216, |
| "bf16_mass_sum": 3.4531892433099847, |
| "quant_mean_mass": 0.0006816201293818956, |
| "bf16_mean_mass": 0.000677494456211494, |
| "mass_ratio": 1.0060896043245462, |
| "mass_ratio_ci95": [ |
| 1.000000536757677, |
| 1.014290878372139 |
| ], |
| "constant_hazard_predicted_length_ratio": 0.9939472545006222, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 816, |
| "n_conversations": 3, |
| "mean": 2.1070730352544176e-06, |
| "ci95": [ |
| -8.846358509926075e-10, |
| 1.3082898780915662e-05 |
| ], |
| "quant_mass_sum": 2.9794942109217684, |
| "bf16_mass_sum": 2.9777748393250008, |
| "quant_mean_mass": 0.003651340944757069, |
| "bf16_mean_mass": 0.0036492338717218145, |
| "mass_ratio": 1.0005774014791384, |
| "mass_ratio_ci95": [ |
| 0.9999998783866034, |
| 1.0017079536361635 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 5097, |
| "n_conversations": 3, |
| "mean": -3.02161450684235e-05, |
| "ci95": [ |
| -5.8206698305767504e-05, |
| 0.00012899518386113673 |
| ], |
| "quant_mass_sum": 11.02281135870327, |
| "bf16_mass_sum": 11.176823050117026, |
| "quant_mean_mass": 0.002162607682696345, |
| "bf16_mean_mass": 0.0021928238277647686, |
| "mass_ratio": 0.9862204411107552, |
| "mass_ratio_ci95": [ |
| 0.9767089183879517, |
| 1.0423326102755763 |
| ] |
| } |
| } |
| }, |
| "math": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 71, |
| "n_conversations": 71, |
| "mean": -0.005524668880630876, |
| "ci95": [ |
| -0.014559900743049607, |
| 0.0016335276630332187 |
| ] |
| }, |
| "eos": { |
| "n_positions": 81, |
| "n_conversations": 81, |
| "mean": 0.0034573550984580174, |
| "ci95": [ |
| -0.0005783126379559387, |
| 0.008438537630162139 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 81, |
| "n_conversations": 81, |
| "mean": -0.008781633019827617, |
| "ci95": [ |
| -0.02303180942303031, |
| 0.0033931781932653345 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 74, |
| "n_conversations": 21, |
| "mean": -0.019866013322887344, |
| "ci95": [ |
| -0.0427351496562534, |
| 0.004635147730529196 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 92629, |
| "n_conversations": 71, |
| "mean": -7.611240476963172e-07, |
| "ci95": [ |
| -6.721258880523155e-06, |
| 5.026061196906248e-06 |
| ], |
| "quant_mass_sum": 72.25979381970194, |
| "bf16_mass_sum": 72.33029597911602, |
| "quant_mean_mass": 0.0007800990383109171, |
| "bf16_mean_mass": 0.0007808601623586136, |
| "mass_ratio": 0.9990252748387144, |
| "mass_ratio_ci95": [ |
| 0.9927953375258589, |
| 1.006220296569122 |
| ], |
| "constant_hazard_predicted_length_ratio": 1.000975676177405, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 22608, |
| "n_conversations": 81, |
| "mean": 9.178981904381253e-06, |
| "ci95": [ |
| -1.1871175103322098e-05, |
| 2.817704682477872e-05 |
| ], |
| "quant_mass_sum": 81.78174310467487, |
| "bf16_mass_sum": 81.57422468178063, |
| "quant_mean_mass": 0.0036173807105747907, |
| "bf16_mean_mass": 0.00360820172867041, |
| "mass_ratio": 1.0025439214862755, |
| "mass_ratio_ci95": [ |
| 0.9970676548506806, |
| 1.0083215693473286 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 92629, |
| "n_conversations": 71, |
| "mean": -1.0895044483818789e-05, |
| "ci95": [ |
| -4.075010058226018e-05, |
| 3.1112642310803784e-05 |
| ], |
| "quant_mass_sum": 534.2651286587221, |
| "bf16_mass_sum": 535.2743257342138, |
| "quant_mean_mass": 0.005767795492326616, |
| "bf16_mean_mass": 0.005778690536810435, |
| "mass_ratio": 0.9981146170720828, |
| "mass_ratio_ci95": [ |
| 0.9933094257992642, |
| 1.005466825859799 |
| ] |
| } |
| } |
| }, |
| "multilingual": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 8, |
| "n_conversations": 8, |
| "mean": -0.0016825755509941587, |
| "ci95": [ |
| -0.005027927421199685, |
| -2.5326005754777725e-06 |
| ] |
| }, |
| "eos": { |
| "n_positions": 9, |
| "n_conversations": 9, |
| "mean": 0.011190537421333444, |
| "ci95": [ |
| 3.0515736069997646e-05, |
| 0.026581764660599225 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 9, |
| "n_conversations": 9, |
| "mean": -0.02737001913230112, |
| "ci95": [ |
| -0.11342743290917698, |
| 0.04470726878572515 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 4, |
| "n_conversations": 1, |
| "mean": 0.043029382824897766, |
| "ci95": [ |
| 0.043029382824897766, |
| 0.043029382824897766 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 35540, |
| "n_conversations": 8, |
| "mean": 2.806991808990477e-07, |
| "ci95": [ |
| -8.47766208328651e-07, |
| 1.4299227183429116e-06 |
| ], |
| "quant_mass_sum": 8.129669166233672, |
| "bf16_mass_sum": 8.119693117344518, |
| "quant_mean_mass": 0.00022874702212250062, |
| "bf16_mean_mass": 0.00022846632294160152, |
| "mass_ratio": 1.0012286238833146, |
| "mass_ratio_ci95": [ |
| 0.9964960832788375, |
| 1.005807567010007 |
| ], |
| "constant_hazard_predicted_length_ratio": 0.9987728837809797, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 12081, |
| "n_conversations": 9, |
| "mean": 1.4856586100885415e-05, |
| "ci95": [ |
| 5.466718175252175e-06, |
| 3.866984354744618e-05 |
| ], |
| "quant_mass_sum": 8.704175032754483, |
| "bf16_mass_sum": 8.524692616069686, |
| "quant_mean_mass": 0.000720484648022058, |
| "bf16_mean_mass": 0.0007056280619211725, |
| "mass_ratio": 1.021054415070223, |
| "mass_ratio_ci95": [ |
| 1.0082205478318014, |
| 1.0347783854868693 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 35540, |
| "n_conversations": 8, |
| "mean": -1.0140279890315519e-05, |
| "ci95": [ |
| -4.24212554086613e-05, |
| 2.749702458484906e-05 |
| ], |
| "quant_mass_sum": 139.0185503541471, |
| "bf16_mass_sum": 139.37893590144893, |
| "quant_mean_mass": 0.003911608057235428, |
| "bf16_mean_mass": 0.003921748337125743, |
| "mass_ratio": 0.9974143471180132, |
| "mass_ratio_ci95": [ |
| 0.9883745690726624, |
| 1.0076596676517873 |
| ] |
| } |
| } |
| }, |
| "stem": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 4, |
| "n_conversations": 4, |
| "mean": 0.00017581030454039137, |
| "ci95": [ |
| -1.1920542419829872e-07, |
| 0.0005275799212540733 |
| ] |
| }, |
| "eos": { |
| "n_positions": 4, |
| "n_conversations": 4, |
| "mean": -1.2097992566850735e-05, |
| "ci95": [ |
| -2.771260915324092e-05, |
| 3.5166240195394494e-06 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 4, |
| "n_conversations": 4, |
| "mean": -0.007828249246813357, |
| "ci95": [ |
| -0.016767692752182484, |
| -4.063849337399006e-05 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 1, |
| "n_conversations": 1, |
| "mean": 0.01944941282272339, |
| "ci95": [ |
| 0.01944941282272339, |
| 0.01944941282272339 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 15859, |
| "n_conversations": 4, |
| "mean": -6.208561233543981e-06, |
| "ci95": [ |
| -2.596697860518136e-05, |
| 1.6716225866924525e-08 |
| ], |
| "quant_mass_sum": 4.224383705906368, |
| "bf16_mass_sum": 4.322845278509141, |
| "quant_mean_mass": 0.0002663713794001115, |
| "bf16_mean_mass": 0.0002725799406336554, |
| "mass_ratio": 0.9772229709232779, |
| "mass_ratio_ci95": [ |
| 0.9404859746882553, |
| 1.0001018239024269 |
| ], |
| "constant_hazard_predicted_length_ratio": 1.0233079141142194, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 13964, |
| "n_conversations": 4, |
| "mean": 9.131410400350527e-08, |
| "ci95": [ |
| 1.7610645745720758e-09, |
| 1.8223418209391135e-07 |
| ], |
| "quant_mass_sum": 4.78715794424999, |
| "bf16_mass_sum": 4.785882834101685, |
| "quant_mean_mass": 0.00034282139388785375, |
| "bf16_mean_mass": 0.00034273007978385023, |
| "mass_ratio": 1.0002664315430414, |
| "mass_ratio_ci95": [ |
| 1.0000050633611948, |
| 1.000907533723477 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 15859, |
| "n_conversations": 4, |
| "mean": 7.442076070004216e-06, |
| "ci95": [ |
| -5.3631179997768144e-05, |
| 4.465379025085888e-05 |
| ], |
| "quant_mass_sum": 49.19159357334727, |
| "bf16_mass_sum": 49.07356968895308, |
| "quant_mean_mass": 0.0031018092927263554, |
| "bf16_mean_mass": 0.0030943672166563515, |
| "mass_ratio": 1.0024050397218354, |
| "mass_ratio_ci95": [ |
| 0.9838446820232397, |
| 1.0142883660695177 |
| ] |
| } |
| } |
| }, |
| "structured": { |
| "decision_logp_bias": { |
| "think_close": { |
| "n_positions": 15, |
| "n_conversations": 15, |
| "mean": -0.0036670741124188075, |
| "ci95": [ |
| -0.00894192645434373, |
| 2.393732181473023e-05 |
| ] |
| }, |
| "eos": { |
| "n_positions": 16, |
| "n_conversations": 16, |
| "mean": 0.003804981863282819, |
| "ci95": [ |
| -5.206383942599757e-05, |
| 0.011486516883124232 |
| ] |
| }, |
| "tool_call_close": { |
| "n_positions": 29, |
| "n_conversations": 12, |
| "mean": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "answer_start": { |
| "n_positions": 16, |
| "n_conversations": 16, |
| "mean": 0.004904084223060767, |
| "ci95": [ |
| -1.206430710820605e-06, |
| 0.014640527869820185 |
| ] |
| }, |
| "reverification_start": { |
| "n_positions": 12, |
| "n_conversations": 8, |
| "mean": -0.02378308823487411, |
| "ci95": [ |
| -0.06273738747196538, |
| 0.008505975988086771 |
| ] |
| } |
| }, |
| "hazard_bias": { |
| "think_close": { |
| "n_positions": 26106, |
| "n_conversations": 15, |
| "mean": 1.4806849459940611e-06, |
| "ci95": [ |
| -1.0638009944021218e-05, |
| 1.3964512994594026e-05 |
| ], |
| "quant_mass_sum": 15.58094347264863, |
| "bf16_mass_sum": 15.542288711448506, |
| "quant_mean_mass": 0.0005968338111027591, |
| "bf16_mean_mass": 0.000595353126156765, |
| "mass_ratio": 1.0024870700781443, |
| "mass_ratio_ci95": [ |
| 0.986802831700198, |
| 1.0198599165603561 |
| ], |
| "constant_hazard_predicted_length_ratio": 0.9975191000937793, |
| "constant_hazard_caveat": "Descriptive inverse mean-hazard proxy; teacher-forced/off-policy contexts and nonconstant hazards do not establish generated length." |
| }, |
| "eos_in_answer": { |
| "n_positions": 7856, |
| "n_conversations": 16, |
| "mean": 6.011998117328673e-06, |
| "ci95": [ |
| -8.371951459471121e-08, |
| 3.5570277610541766e-05 |
| ], |
| "quant_mass_sum": 15.70943008100527, |
| "bf16_mass_sum": 15.662199823795536, |
| "quant_mean_mass": 0.0019996728718183897, |
| "bf16_mean_mass": 0.001993660873701061, |
| "mass_ratio": 1.003015557057188, |
| "mass_ratio_ci95": [ |
| 0.9999627496263969, |
| 1.009339442724538 |
| ] |
| }, |
| "tool_close": { |
| "n_positions": 3025, |
| "n_conversations": 12, |
| "mean": -2.835492722162358e-13, |
| "ci95": [ |
| -2.1534758151175124e-11, |
| 2.0893414477950814e-11 |
| ], |
| "quant_mass_sum": 29.000000757366035, |
| "bf16_mass_sum": 29.00000075822377, |
| "quant_mean_mass": 0.00958677710987307, |
| "bf16_mean_mass": 0.009586777110156618, |
| "mass_ratio": 0.9999999999704229, |
| "mass_ratio_ci95": [ |
| 0.9999999977126025, |
| 1.000000001980899 |
| ] |
| }, |
| "reverification_lexical_start": { |
| "n_positions": 26106, |
| "n_conversations": 15, |
| "mean": 4.1353217797483426e-05, |
| "ci95": [ |
| 7.417139212354176e-06, |
| 0.00010182338573841413 |
| ], |
| "quant_mass_sum": 137.08770498503694, |
| "bf16_mass_sum": 136.00813788121584, |
| "quant_mean_mass": 0.00525119531851057, |
| "bf16_mean_mass": 0.005209842100713086, |
| "mass_ratio": 1.0079375184502852, |
| "mass_ratio_ci95": [ |
| 1.0013573524438273, |
| 1.0157667118061582 |
| ] |
| } |
| } |
| } |
| } |
| }, |
| "matched_short30": [ |
| { |
| "artifact": "bf16", |
| "trials": 90, |
| "passed": 86, |
| "complete": true, |
| "accuracy_vs_matched_bf16": { |
| "agentic_coding_on": { |
| "n_pairs": 90, |
| "n_prompts": 30, |
| "paired_difference": 0.0, |
| "ci95_prompt_bootstrap": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| "length_vs_matched_bf16": { |
| "n_prompts": 30, |
| "n_pairs": 90, |
| "ci": "95% paired prompt-cluster bootstrap, 4000 draws", |
| "minimum_paired_seeds_per_prompt": 3, |
| "three_seed_coverage_complete": true, |
| "total_tokens": { |
| "ratio_of_sums": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "quant_sum": 218843, |
| "bf16_sum": 218843 |
| }, |
| "thinking_tokens": { |
| "ratio_of_sums": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "quant_sum": 139299, |
| "bf16_sum": 139299 |
| }, |
| "unfinished": { |
| "rate": 0.07777777777777778, |
| "bf16_rate": 0.07777777777777778, |
| "paired_difference": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "loop": { |
| "rate": 0.044444444444444446, |
| "bf16_rate": 0.044444444444444446, |
| "paired_difference": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "per_prompt": [ |
| { |
| "id": "coding-01-stable-unique-1", |
| "total_tokens": { |
| "quant": [ |
| 469, |
| 662, |
| 688 |
| ], |
| "bf16": [ |
| 469, |
| 662, |
| 688 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 174, |
| 289, |
| 330 |
| ], |
| "bf16": [ |
| 174, |
| 289, |
| 330 |
| ] |
| } |
| }, |
| { |
| "id": "coding-02-half-open-intervals-1", |
| "total_tokens": { |
| "quant": [ |
| 1444, |
| 2539, |
| 2101 |
| ], |
| "bf16": [ |
| 1444, |
| 2539, |
| 2101 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 788, |
| 1817, |
| 1289 |
| ], |
| "bf16": [ |
| 788, |
| 1817, |
| 1289 |
| ] |
| } |
| }, |
| { |
| "id": "coding-03-path-cycle-flatten-1", |
| "total_tokens": { |
| "quant": [ |
| 6804, |
| 5947, |
| 5919 |
| ], |
| "bf16": [ |
| 6804, |
| 5947, |
| 5919 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 3137, |
| 4928, |
| 4856 |
| ], |
| "bf16": [ |
| 3137, |
| 4928, |
| 4856 |
| ] |
| } |
| }, |
| { |
| "id": "coding-04-rotate-list-1", |
| "total_tokens": { |
| "quant": [ |
| 702, |
| 1247, |
| 514 |
| ], |
| "bf16": [ |
| 702, |
| 1247, |
| 514 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 290, |
| 845, |
| 231 |
| ], |
| "bf16": [ |
| 290, |
| 845, |
| 231 |
| ] |
| } |
| }, |
| { |
| "id": "coding-05-fixed-chunks-1", |
| "total_tokens": { |
| "quant": [ |
| 1782, |
| 969, |
| 1284 |
| ], |
| "bf16": [ |
| 1782, |
| 969, |
| 1284 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1261, |
| 415, |
| 437 |
| ], |
| "bf16": [ |
| 1261, |
| 415, |
| 437 |
| ] |
| } |
| }, |
| { |
| "id": "coding-06-duration-parser-1", |
| "total_tokens": { |
| "quant": [ |
| 2024, |
| 2361, |
| 5483 |
| ], |
| "bf16": [ |
| 2024, |
| 2361, |
| 5483 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1250, |
| 1642, |
| 3876 |
| ], |
| "bf16": [ |
| 1250, |
| 1642, |
| 3876 |
| ] |
| } |
| }, |
| { |
| "id": "coding-07-byte-safe-truncation-1", |
| "total_tokens": { |
| "quant": [ |
| 922, |
| 981, |
| 3576 |
| ], |
| "bf16": [ |
| 922, |
| 981, |
| 3576 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 477, |
| 569, |
| 2588 |
| ], |
| "bf16": [ |
| 477, |
| 569, |
| 2588 |
| ] |
| } |
| }, |
| { |
| "id": "coding-08-deep-merge-1", |
| "total_tokens": { |
| "quant": [ |
| 1619, |
| 1812, |
| 2668 |
| ], |
| "bf16": [ |
| 1619, |
| 1812, |
| 2668 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 910, |
| 1201, |
| 1949 |
| ], |
| "bf16": [ |
| 910, |
| 1201, |
| 1949 |
| ] |
| } |
| }, |
| { |
| "id": "coding-09-nested-lookup-1", |
| "total_tokens": { |
| "quant": [ |
| 2349, |
| 4705, |
| 6976 |
| ], |
| "bf16": [ |
| 2349, |
| 4705, |
| 6976 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1643, |
| 3787, |
| 3401 |
| ], |
| "bf16": [ |
| 1643, |
| 3787, |
| 3401 |
| ] |
| } |
| }, |
| { |
| "id": "coding-10-sliding-window-1", |
| "total_tokens": { |
| "quant": [ |
| 956, |
| 573, |
| 1171 |
| ], |
| "bf16": [ |
| 956, |
| 573, |
| 1171 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 432, |
| 196, |
| 642 |
| ], |
| "bf16": [ |
| 432, |
| 196, |
| 642 |
| ] |
| } |
| }, |
| { |
| "id": "coding-11-lru-cache-1", |
| "total_tokens": { |
| "quant": [ |
| 1200, |
| 1151, |
| 1575 |
| ], |
| "bf16": [ |
| 1200, |
| 1151, |
| 1575 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 452, |
| 264, |
| 869 |
| ], |
| "bf16": [ |
| 452, |
| 264, |
| 869 |
| ] |
| } |
| }, |
| { |
| "id": "coding-12-ttl-cache-1", |
| "total_tokens": { |
| "quant": [ |
| 1170, |
| 2723, |
| 3288 |
| ], |
| "bf16": [ |
| 1170, |
| 2723, |
| 3288 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 496, |
| 1124, |
| 1297 |
| ], |
| "bf16": [ |
| 496, |
| 1124, |
| 1297 |
| ] |
| } |
| }, |
| { |
| "id": "coding-13-query-multimap-1", |
| "total_tokens": { |
| "quant": [ |
| 1714, |
| 3755, |
| 7121 |
| ], |
| "bf16": [ |
| 1714, |
| 3755, |
| 7121 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 993, |
| 2806, |
| 5647 |
| ], |
| "bf16": [ |
| 993, |
| 2806, |
| 5647 |
| ] |
| } |
| }, |
| { |
| "id": "coding-14-absolute-path-normalizer-1", |
| "total_tokens": { |
| "quant": [ |
| 1984, |
| 1715, |
| 2256 |
| ], |
| "bf16": [ |
| 1984, |
| 1715, |
| 2256 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1334, |
| 1183, |
| 1533 |
| ], |
| "bf16": [ |
| 1334, |
| 1183, |
| 1533 |
| ] |
| } |
| }, |
| { |
| "id": "coding-15-strict-booleans-1", |
| "total_tokens": { |
| "quant": [ |
| 895, |
| 1215, |
| 2004 |
| ], |
| "bf16": [ |
| 895, |
| 1215, |
| 2004 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 341, |
| 532, |
| 1119 |
| ], |
| "bf16": [ |
| 341, |
| 532, |
| 1119 |
| ] |
| } |
| }, |
| { |
| "id": "coding-16-recursive-redaction-1", |
| "total_tokens": { |
| "quant": [ |
| 1214, |
| 1184, |
| 1537 |
| ], |
| "bf16": [ |
| 1214, |
| 1184, |
| 1537 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 490, |
| 519, |
| 877 |
| ], |
| "bf16": [ |
| 490, |
| 519, |
| 877 |
| ] |
| } |
| }, |
| { |
| "id": "coding-17-csv-dictionaries-1", |
| "total_tokens": { |
| "quant": [ |
| 6199, |
| 7001, |
| 6536 |
| ], |
| "bf16": [ |
| 6199, |
| 7001, |
| 6536 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 5587, |
| 3574, |
| 5488 |
| ], |
| "bf16": [ |
| 5587, |
| 3574, |
| 5488 |
| ] |
| } |
| }, |
| { |
| "id": "coding-18-header-multimap-1", |
| "total_tokens": { |
| "quant": [ |
| 3202, |
| 965, |
| 717 |
| ], |
| "bf16": [ |
| 3202, |
| 965, |
| 717 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 2505, |
| 482, |
| 210 |
| ], |
| "bf16": [ |
| 2505, |
| 482, |
| 210 |
| ] |
| } |
| }, |
| { |
| "id": "coding-19-retry-boundaries-1", |
| "total_tokens": { |
| "quant": [ |
| 1087, |
| 2436, |
| 1657 |
| ], |
| "bf16": [ |
| 1087, |
| 2436, |
| 1657 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 524, |
| 1258, |
| 986 |
| ], |
| "bf16": [ |
| 524, |
| 1258, |
| 986 |
| ] |
| } |
| }, |
| { |
| "id": "coding-20-memoize-none-1", |
| "total_tokens": { |
| "quant": [ |
| 676, |
| 954, |
| 1874 |
| ], |
| "bf16": [ |
| 676, |
| 954, |
| 1874 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 240, |
| 415, |
| 1109 |
| ], |
| "bf16": [ |
| 240, |
| 415, |
| 1109 |
| ] |
| } |
| }, |
| { |
| "id": "coding-21-topological-order-1", |
| "total_tokens": { |
| "quant": [ |
| 1747, |
| 2463, |
| 3367 |
| ], |
| "bf16": [ |
| 1747, |
| 2463, |
| 3367 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 967, |
| 1410, |
| 2281 |
| ], |
| "bf16": [ |
| 967, |
| 1410, |
| 2281 |
| ] |
| } |
| }, |
| { |
| "id": "coding-22-shortest-path-1", |
| "total_tokens": { |
| "quant": [ |
| 1231, |
| 1278, |
| 1405 |
| ], |
| "bf16": [ |
| 1231, |
| 1278, |
| 1405 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 557, |
| 639, |
| 766 |
| ], |
| "bf16": [ |
| 557, |
| 639, |
| 766 |
| ] |
| } |
| }, |
| { |
| "id": "coding-23-inclusive-ranges-1", |
| "total_tokens": { |
| "quant": [ |
| 3860, |
| 3765, |
| 8192 |
| ], |
| "bf16": [ |
| 3860, |
| 3765, |
| 8192 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 2867, |
| 2821, |
| 8128 |
| ], |
| "bf16": [ |
| 2867, |
| 2821, |
| 8128 |
| ] |
| } |
| }, |
| { |
| "id": "coding-24-cursor-pagination-1", |
| "total_tokens": { |
| "quant": [ |
| 4939, |
| 3797, |
| 5074 |
| ], |
| "bf16": [ |
| 4939, |
| 3797, |
| 5074 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 3244, |
| 2657, |
| 3909 |
| ], |
| "bf16": [ |
| 3244, |
| 2657, |
| 3909 |
| ] |
| } |
| }, |
| { |
| "id": "coding-25-ring-buffer-1", |
| "total_tokens": { |
| "quant": [ |
| 1619, |
| 758, |
| 1606 |
| ], |
| "bf16": [ |
| 1619, |
| 758, |
| 1606 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 848, |
| 174, |
| 637 |
| ], |
| "bf16": [ |
| 848, |
| 174, |
| 637 |
| ] |
| } |
| }, |
| { |
| "id": "coding-26-balanced-delimiters-1", |
| "total_tokens": { |
| "quant": [ |
| 1659, |
| 1758, |
| 1915 |
| ], |
| "bf16": [ |
| 1659, |
| 1758, |
| 1915 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 916, |
| 960, |
| 1190 |
| ], |
| "bf16": [ |
| 916, |
| 960, |
| 1190 |
| ] |
| } |
| }, |
| { |
| "id": "coding-27-run-length-codec-1", |
| "total_tokens": { |
| "quant": [ |
| 708, |
| 478, |
| 5144 |
| ], |
| "bf16": [ |
| 708, |
| 478, |
| 5144 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 243, |
| 119, |
| 3837 |
| ], |
| "bf16": [ |
| 243, |
| 119, |
| 3837 |
| ] |
| } |
| }, |
| { |
| "id": "coding-28-escaped-split-1", |
| "total_tokens": { |
| "quant": [ |
| 1319, |
| 1548, |
| 1796 |
| ], |
| "bf16": [ |
| 1319, |
| 1548, |
| 1796 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 731, |
| 847, |
| 1088 |
| ], |
| "bf16": [ |
| 731, |
| 847, |
| 1088 |
| ] |
| } |
| }, |
| { |
| "id": "coding-29-numeric-version-order-1", |
| "total_tokens": { |
| "quant": [ |
| 1870, |
| 1736, |
| 4921 |
| ], |
| "bf16": [ |
| 1870, |
| 1736, |
| 4921 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1074, |
| 915, |
| 1006 |
| ], |
| "bf16": [ |
| 1074, |
| 915, |
| 1006 |
| ] |
| } |
| }, |
| { |
| "id": "coding-30-recursive-diff-1", |
| "total_tokens": { |
| "quant": [ |
| 1648, |
| 2628, |
| 2362 |
| ], |
| "bf16": [ |
| 1648, |
| 2628, |
| 2362 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 933, |
| 1951, |
| 1680 |
| ], |
| "bf16": [ |
| 933, |
| 1951, |
| 1680 |
| ] |
| } |
| } |
| ] |
| } |
| }, |
| { |
| "artifact": "fp8-block-calib-v1", |
| "trials": 90, |
| "passed": 82, |
| "complete": true, |
| "accuracy_vs_matched_bf16": { |
| "agentic_coding_on": { |
| "n_pairs": 90, |
| "n_prompts": 30, |
| "paired_difference": -0.04444444444444444, |
| "ci95_prompt_bootstrap": [ |
| -0.11111111111111112, |
| 0.02222222222222222 |
| ] |
| } |
| }, |
| "length_vs_matched_bf16": { |
| "n_prompts": 30, |
| "n_pairs": 90, |
| "ci": "95% paired prompt-cluster bootstrap, 4000 draws", |
| "minimum_paired_seeds_per_prompt": 3, |
| "three_seed_coverage_complete": true, |
| "total_tokens": { |
| "ratio_of_sums": 1.0476094734581411, |
| "ci95": [ |
| 0.9169603224543987, |
| 1.2115954758852716 |
| ], |
| "quant_sum": 229262, |
| "bf16_sum": 218843 |
| }, |
| "thinking_tokens": { |
| "ratio_of_sums": 1.1279621533535775, |
| "ci95": [ |
| 0.9371457429405465, |
| 1.3763876786289677 |
| ], |
| "quant_sum": 157124, |
| "bf16_sum": 139299 |
| }, |
| "unfinished": { |
| "rate": 0.06666666666666667, |
| "bf16_rate": 0.07777777777777778, |
| "paired_difference": -0.01111111111111111, |
| "ci95": [ |
| -0.04444444444444444, |
| 0.02222222222222222 |
| ] |
| }, |
| "loop": { |
| "rate": 0.05555555555555555, |
| "bf16_rate": 0.044444444444444446, |
| "paired_difference": 0.01111111111111111, |
| "ci95": [ |
| -0.05555555555555555, |
| 0.08888888888888888 |
| ] |
| }, |
| "per_prompt": [ |
| { |
| "id": "coding-01-stable-unique-1", |
| "total_tokens": { |
| "quant": [ |
| 427, |
| 940, |
| 5628 |
| ], |
| "bf16": [ |
| 662, |
| 469, |
| 688 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 123, |
| 527, |
| 5254 |
| ], |
| "bf16": [ |
| 289, |
| 174, |
| 330 |
| ] |
| } |
| }, |
| { |
| "id": "coding-02-half-open-intervals-1", |
| "total_tokens": { |
| "quant": [ |
| 1439, |
| 2179, |
| 2213 |
| ], |
| "bf16": [ |
| 1444, |
| 2539, |
| 2101 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 797, |
| 985, |
| 1221 |
| ], |
| "bf16": [ |
| 788, |
| 1817, |
| 1289 |
| ] |
| } |
| }, |
| { |
| "id": "coding-03-path-cycle-flatten-1", |
| "total_tokens": { |
| "quant": [ |
| 3169, |
| 3351, |
| 8192 |
| ], |
| "bf16": [ |
| 6804, |
| 5947, |
| 5919 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 2468, |
| 2502, |
| 4338 |
| ], |
| "bf16": [ |
| 3137, |
| 4928, |
| 4856 |
| ] |
| } |
| }, |
| { |
| "id": "coding-04-rotate-list-1", |
| "total_tokens": { |
| "quant": [ |
| 504, |
| 641, |
| 713 |
| ], |
| "bf16": [ |
| 702, |
| 1247, |
| 514 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 194, |
| 316, |
| 419 |
| ], |
| "bf16": [ |
| 290, |
| 845, |
| 231 |
| ] |
| } |
| }, |
| { |
| "id": "coding-05-fixed-chunks-1", |
| "total_tokens": { |
| "quant": [ |
| 1052, |
| 1031, |
| 1641 |
| ], |
| "bf16": [ |
| 1782, |
| 1284, |
| 969 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 567, |
| 562, |
| 1112 |
| ], |
| "bf16": [ |
| 1261, |
| 437, |
| 415 |
| ] |
| } |
| }, |
| { |
| "id": "coding-06-duration-parser-1", |
| "total_tokens": { |
| "quant": [ |
| 1044, |
| 2057, |
| 6108 |
| ], |
| "bf16": [ |
| 2024, |
| 5483, |
| 2361 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 500, |
| 1400, |
| 4426 |
| ], |
| "bf16": [ |
| 1250, |
| 3876, |
| 1642 |
| ] |
| } |
| }, |
| { |
| "id": "coding-07-byte-safe-truncation-1", |
| "total_tokens": { |
| "quant": [ |
| 1051, |
| 1284, |
| 2011 |
| ], |
| "bf16": [ |
| 3576, |
| 981, |
| 922 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 554, |
| 837, |
| 1476 |
| ], |
| "bf16": [ |
| 2588, |
| 569, |
| 477 |
| ] |
| } |
| }, |
| { |
| "id": "coding-08-deep-merge-1", |
| "total_tokens": { |
| "quant": [ |
| 1785, |
| 3116, |
| 8192 |
| ], |
| "bf16": [ |
| 2668, |
| 1812, |
| 1619 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1160, |
| 2404, |
| 8128 |
| ], |
| "bf16": [ |
| 1949, |
| 1201, |
| 910 |
| ] |
| } |
| }, |
| { |
| "id": "coding-09-nested-lookup-1", |
| "total_tokens": { |
| "quant": [ |
| 1962, |
| 2833, |
| 2635 |
| ], |
| "bf16": [ |
| 6976, |
| 4705, |
| 2349 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1257, |
| 1972, |
| 1858 |
| ], |
| "bf16": [ |
| 3401, |
| 3787, |
| 1643 |
| ] |
| } |
| }, |
| { |
| "id": "coding-10-sliding-window-1", |
| "total_tokens": { |
| "quant": [ |
| 469, |
| 1033, |
| 1056 |
| ], |
| "bf16": [ |
| 956, |
| 573, |
| 1171 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 178, |
| 334, |
| 510 |
| ], |
| "bf16": [ |
| 432, |
| 196, |
| 642 |
| ] |
| } |
| }, |
| { |
| "id": "coding-11-lru-cache-1", |
| "total_tokens": { |
| "quant": [ |
| 1044, |
| 1020, |
| 1886 |
| ], |
| "bf16": [ |
| 1151, |
| 1575, |
| 1200 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 201, |
| 278, |
| 1014 |
| ], |
| "bf16": [ |
| 264, |
| 869, |
| 452 |
| ] |
| } |
| }, |
| { |
| "id": "coding-12-ttl-cache-1", |
| "total_tokens": { |
| "quant": [ |
| 1763, |
| 1298, |
| 2450 |
| ], |
| "bf16": [ |
| 1170, |
| 3288, |
| 2723 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1064, |
| 636, |
| 1169 |
| ], |
| "bf16": [ |
| 496, |
| 1297, |
| 1124 |
| ] |
| } |
| }, |
| { |
| "id": "coding-13-query-multimap-1", |
| "total_tokens": { |
| "quant": [ |
| 2971, |
| 4017, |
| 7304 |
| ], |
| "bf16": [ |
| 7121, |
| 1714, |
| 3755 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 2220, |
| 2726, |
| 6300 |
| ], |
| "bf16": [ |
| 5647, |
| 993, |
| 2806 |
| ] |
| } |
| }, |
| { |
| "id": "coding-14-absolute-path-normalizer-1", |
| "total_tokens": { |
| "quant": [ |
| 2228, |
| 1439, |
| 2031 |
| ], |
| "bf16": [ |
| 1984, |
| 1715, |
| 2256 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1572, |
| 672, |
| 1267 |
| ], |
| "bf16": [ |
| 1334, |
| 1183, |
| 1533 |
| ] |
| } |
| }, |
| { |
| "id": "coding-15-strict-booleans-1", |
| "total_tokens": { |
| "quant": [ |
| 1911, |
| 1174, |
| 910 |
| ], |
| "bf16": [ |
| 895, |
| 1215, |
| 2004 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1119, |
| 537, |
| 339 |
| ], |
| "bf16": [ |
| 341, |
| 532, |
| 1119 |
| ] |
| } |
| }, |
| { |
| "id": "coding-16-recursive-redaction-1", |
| "total_tokens": { |
| "quant": [ |
| 1353, |
| 1700, |
| 5177 |
| ], |
| "bf16": [ |
| 1214, |
| 1537, |
| 1184 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 764, |
| 1017, |
| 3548 |
| ], |
| "bf16": [ |
| 490, |
| 877, |
| 519 |
| ] |
| } |
| }, |
| { |
| "id": "coding-17-csv-dictionaries-1", |
| "total_tokens": { |
| "quant": [ |
| 8192, |
| 8192, |
| 8192 |
| ], |
| "bf16": [ |
| 6199, |
| 7001, |
| 6536 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 8128, |
| 8128, |
| 6226 |
| ], |
| "bf16": [ |
| 5587, |
| 3574, |
| 5488 |
| ] |
| } |
| }, |
| { |
| "id": "coding-18-header-multimap-1", |
| "total_tokens": { |
| "quant": [ |
| 5012, |
| 1847, |
| 1596 |
| ], |
| "bf16": [ |
| 3202, |
| 965, |
| 717 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 4446, |
| 1312, |
| 1071 |
| ], |
| "bf16": [ |
| 2505, |
| 482, |
| 210 |
| ] |
| } |
| }, |
| { |
| "id": "coding-19-retry-boundaries-1", |
| "total_tokens": { |
| "quant": [ |
| 1238, |
| 3269, |
| 3235 |
| ], |
| "bf16": [ |
| 2436, |
| 1087, |
| 1657 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 637, |
| 2206, |
| 1768 |
| ], |
| "bf16": [ |
| 1258, |
| 524, |
| 986 |
| ] |
| } |
| }, |
| { |
| "id": "coding-20-memoize-none-1", |
| "total_tokens": { |
| "quant": [ |
| 996, |
| 1341, |
| 1553 |
| ], |
| "bf16": [ |
| 676, |
| 954, |
| 1874 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 450, |
| 711, |
| 928 |
| ], |
| "bf16": [ |
| 240, |
| 415, |
| 1109 |
| ] |
| } |
| }, |
| { |
| "id": "coding-21-topological-order-1", |
| "total_tokens": { |
| "quant": [ |
| 2560, |
| 2421, |
| 1990 |
| ], |
| "bf16": [ |
| 3367, |
| 1747, |
| 2463 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1685, |
| 1537, |
| 1042 |
| ], |
| "bf16": [ |
| 2281, |
| 967, |
| 1410 |
| ] |
| } |
| }, |
| { |
| "id": "coding-22-shortest-path-1", |
| "total_tokens": { |
| "quant": [ |
| 1191, |
| 1751, |
| 2572 |
| ], |
| "bf16": [ |
| 1231, |
| 1278, |
| 1405 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 616, |
| 714, |
| 1851 |
| ], |
| "bf16": [ |
| 557, |
| 639, |
| 766 |
| ] |
| } |
| }, |
| { |
| "id": "coding-23-inclusive-ranges-1", |
| "total_tokens": { |
| "quant": [ |
| 2418, |
| 4452, |
| 8192 |
| ], |
| "bf16": [ |
| 3860, |
| 3765, |
| 8192 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1443, |
| 2870, |
| 8128 |
| ], |
| "bf16": [ |
| 2867, |
| 2821, |
| 8128 |
| ] |
| } |
| }, |
| { |
| "id": "coding-24-cursor-pagination-1", |
| "total_tokens": { |
| "quant": [ |
| 1864, |
| 2201, |
| 6452 |
| ], |
| "bf16": [ |
| 4939, |
| 5074, |
| 3797 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1215, |
| 1331, |
| 4152 |
| ], |
| "bf16": [ |
| 3244, |
| 3909, |
| 2657 |
| ] |
| } |
| }, |
| { |
| "id": "coding-25-ring-buffer-1", |
| "total_tokens": { |
| "quant": [ |
| 1619, |
| 3862, |
| 1714 |
| ], |
| "bf16": [ |
| 1606, |
| 1619, |
| 758 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 435, |
| 1402, |
| 983 |
| ], |
| "bf16": [ |
| 637, |
| 848, |
| 174 |
| ] |
| } |
| }, |
| { |
| "id": "coding-26-balanced-delimiters-1", |
| "total_tokens": { |
| "quant": [ |
| 1973, |
| 2126, |
| 2430 |
| ], |
| "bf16": [ |
| 1659, |
| 1758, |
| 1915 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1112, |
| 1347, |
| 1624 |
| ], |
| "bf16": [ |
| 916, |
| 960, |
| 1190 |
| ] |
| } |
| }, |
| { |
| "id": "coding-27-run-length-codec-1", |
| "total_tokens": { |
| "quant": [ |
| 567, |
| 669, |
| 1075 |
| ], |
| "bf16": [ |
| 708, |
| 5144, |
| 478 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 226, |
| 308, |
| 512 |
| ], |
| "bf16": [ |
| 243, |
| 3837, |
| 119 |
| ] |
| } |
| }, |
| { |
| "id": "coding-28-escaped-split-1", |
| "total_tokens": { |
| "quant": [ |
| 1287, |
| 1363, |
| 2089 |
| ], |
| "bf16": [ |
| 1319, |
| 1796, |
| 1548 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 671, |
| 798, |
| 1360 |
| ], |
| "bf16": [ |
| 731, |
| 1088, |
| 847 |
| ] |
| } |
| }, |
| { |
| "id": "coding-29-numeric-version-order-1", |
| "total_tokens": { |
| "quant": [ |
| 1941, |
| 2073, |
| 3403 |
| ], |
| "bf16": [ |
| 1870, |
| 4921, |
| 1736 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 1129, |
| 1345, |
| 1746 |
| ], |
| "bf16": [ |
| 1074, |
| 1006, |
| 915 |
| ] |
| } |
| }, |
| { |
| "id": "coding-30-recursive-diff-1", |
| "total_tokens": { |
| "quant": [ |
| 1496, |
| 3235, |
| 2181 |
| ], |
| "bf16": [ |
| 2628, |
| 1648, |
| 2362 |
| ] |
| }, |
| "thinking_tokens": { |
| "quant": [ |
| 781, |
| 2480, |
| 1448 |
| ], |
| "bf16": [ |
| 1951, |
| 933, |
| 1680 |
| ] |
| } |
| } |
| ] |
| } |
| } |
| ], |
| "default_auto_gate": { |
| "state": "pass", |
| "run": "fp8-block-calib-v1-length", |
| "complete": true, |
| "requested_kv_cache_dtype": "auto", |
| "passed": 4, |
| "n": 4, |
| "evidence": [ |
| { |
| "id": "default-auto-8k", |
| "ok": true, |
| "prompt_tokens": 8259 |
| }, |
| { |
| "id": "default-auto-thinkon", |
| "ok": true, |
| "prompt_tokens": 64 |
| }, |
| { |
| "id": "tool-07-0", |
| "ok": true, |
| "prompt_tokens": null |
| }, |
| { |
| "id": "default-auto-thinkoff", |
| "ok": true, |
| "prompt_tokens": 68 |
| } |
| ] |
| }, |
| "raw_native_smokes": [ |
| { |
| "artifact": "fp8-block-calib-v1", |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "capability": [ |
| 12, |
| 0 |
| ], |
| "seconds": 159.56861901400003, |
| "status": "passed", |
| "runtime_kv_layers": 0, |
| "actual_kv_dtype": "auto", |
| "real_prompt_passes": 6 |
| }, |
| { |
| "artifact": "fp8-block-calib-v1", |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "capability": [ |
| 12, |
| 0 |
| ], |
| "seconds": 135.39686445899997, |
| "status": "passed", |
| "runtime_kv_layers": 0, |
| "actual_kv_dtype": "fp8_e4m3", |
| "real_prompt_passes": 6 |
| } |
| ], |
| "remaining_scoreboard_release_gate": { |
| "state": "pass", |
| "reasons": [ |
| "advisory: unfinished +1/72, CI includes 0; full30 not higher" |
| ], |
| "blocking_reasons": [], |
| "advisories": [ |
| "advisory: unfinished +1/72, CI includes 0; full30 not higher" |
| ], |
| "orchestrator_rulings": [ |
| { |
| "time": "2026-10-25T17:10:00-04:00", |
| "topic": "unfinished rate increase from small core", |
| "ruling": "non-blocking/advisory", |
| "reason_text": "advisory: unfinished +1/72, CI includes 0; full30 not higher", |
| "evidence": { |
| "core": { |
| "quant_unfinished": 1, |
| "bf16_unfinished": 0, |
| "n": 72, |
| "paired_difference_ci95": [ |
| 0, |
| 0.0417 |
| ] |
| }, |
| "full30": { |
| "quant_unfinished": 6, |
| "bf16_unfinished": 7, |
| "n": 90 |
| }, |
| "thinking_stop_mass": { |
| "ratio": 1.000123, |
| "ci95": [ |
| 0.996, |
| 1.004 |
| ] |
| }, |
| "eos_in_answer_mass_ratio": 1.0038, |
| "tool_close_mass_ratio": 1.0 |
| }, |
| "remaining_condition": "Full behaviour coverage and the missing long retrievals must complete successfully. Then state pass, publication_authorized remains false." |
| } |
| ], |
| "publication_authorized": false |
| }, |
| "evidence_sources": { |
| "eval/scoreboard.json": "327be54a4f4cf974c12b45bc00424613faa56b57c71ac8eb52b567ee0bded345", |
| "eval/results/CODING_AUDIT.json": "3dc5f791436b87fc9cc5dad2506f6e4da5ffa4ed95a9876b2d8a87709d5dac82", |
| "vega-serving/RESULTS.md": "ff459c82489058b5dd5e8147d44c582f588d4582cd6a4c60e9a59d16db94c93a", |
| "vega-serving/clean-fp8-speed-summary.json": "60840fafccb7a80a86b426a37050d7b2487237b9e1bd6d8c88360b6d1fcde421", |
| "vega-serving/CLEAN_FP8_SPEED.md": "180a156ec2ce1af53f2ad0ca53f866755758b2bd4a37b4bf6408b72016ca74c5", |
| "shared/baseline-bench/SUMMARY.md": "3d6cb495c1dd1c397db86e085c756463130f160212679762d38428da6c5e94da", |
| "shared/serving-parsers.json": "5facaff72313d534c060da5f30b65601bfb7a58ad009db77ce6f23fa57717e8a" |
| }, |
| "speed_scope": "Headline is exact clean checkpoint auto/BF16 KV; calibrated-KV companion is a separately labeled secondary row", |
| "vega_clean_speed": { |
| "modes": [ |
| { |
| "tag": "clean-auto-bf16kv", |
| "case_count": 18, |
| "kv_actual": "torch.bfloat16", |
| "actual_pool_tokens": 248865, |
| "context_limit_requested": 262144, |
| "max_req_input_len": 248859, |
| "max_bench_input": 248353, |
| "phase_elapsed": { |
| "seconds": 426.75080037117004, |
| "optional": false |
| } |
| }, |
| { |
| "tag": "clean-unit-fp8kv", |
| "case_count": 2, |
| "kv_actual": "torch.float8_e4m3fn", |
| "actual_pool_tokens": 278528, |
| "context_limit_requested": 262144, |
| "max_req_input_len": 262138, |
| "max_bench_input": 250000, |
| "phase_elapsed": { |
| "seconds": 130.89354395866394, |
| "optional": true |
| } |
| } |
| ], |
| "rows": [ |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "warmup-r0", |
| "batch": 1, |
| "input_tokens": 512, |
| "output_tokens": 64, |
| "decode_tok_s": 44.42563213950309, |
| "ttft_s": 0.08525693797855638, |
| "e2e_aggregate_tok_s": 42.56398697869074, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "single-r0", |
| "batch": 1, |
| "input_tokens": 1024, |
| "output_tokens": 1024, |
| "decode_tok_s": 44.211431404259436, |
| "ttft_s": 0.12892672899761237, |
| "e2e_aggregate_tok_s": 44.008924814476444, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "single-r1", |
| "batch": 1, |
| "input_tokens": 1024, |
| "output_tokens": 1024, |
| "decode_tok_s": 44.19584919717605, |
| "ttft_s": 0.13039094099076465, |
| "e2e_aggregate_tok_s": 43.9906984426462, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "single-r2", |
| "batch": 1, |
| "input_tokens": 1024, |
| "output_tokens": 1024, |
| "decode_tok_s": 44.19736415271943, |
| "ttft_s": 0.13153046197840013, |
| "e2e_aggregate_tok_s": 43.99015941613721, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "prefill8k-r0", |
| "batch": 1, |
| "input_tokens": 8192, |
| "output_tokens": 8, |
| "decode_tok_s": 42.47386966671049, |
| "ttft_s": 1.1062289940018672, |
| "e2e_aggregate_tok_s": 6.290850507702435, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "prefill8k-r1", |
| "batch": 1, |
| "input_tokens": 8192, |
| "output_tokens": 8, |
| "decode_tok_s": 42.57185297232053, |
| "ttft_s": 1.102023774990812, |
| "e2e_aggregate_tok_s": 6.314329167135978, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "prefill8k-r2", |
| "batch": 1, |
| "input_tokens": 8192, |
| "output_tokens": 8, |
| "decode_tok_s": 42.453158167223634, |
| "ttft_s": 1.1039147799892817, |
| "e2e_aggregate_tok_s": 6.302456964308019, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "prefill8x8k-r0", |
| "batch": 8, |
| "input_tokens": 8192, |
| "output_tokens": 1, |
| "decode_tok_s": null, |
| "ttft_s": 4.969956030123285, |
| "e2e_aggregate_tok_s": 0.9069364913513717, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "long32k-r0", |
| "batch": 1, |
| "input_tokens": 32768, |
| "output_tokens": 256, |
| "decode_tok_s": 37.33410604248859, |
| "ttft_s": 6.308706900977995, |
| "e2e_aggregate_tok_s": 19.481953815139946, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "long32k-r1", |
| "batch": 1, |
| "input_tokens": 32768, |
| "output_tokens": 256, |
| "decode_tok_s": 37.32367403366498, |
| "ttft_s": 6.251670500001637, |
| "e2e_aggregate_tok_s": 19.563807985908202, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "long128k-r0", |
| "batch": 1, |
| "input_tokens": 131072, |
| "output_tokens": 256, |
| "decode_tok_s": 25.27107056199631, |
| "ttft_s": 54.964571178978076, |
| "e2e_aggregate_tok_s": 3.9348539606519783, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "conc1-r0", |
| "batch": 1, |
| "input_tokens": 1024, |
| "output_tokens": 512, |
| "decode_tok_s": 44.252056899903, |
| "ttft_s": 0.14965118098189123, |
| "e2e_aggregate_tok_s": 43.77043279410766, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "conc4-r0", |
| "batch": 4, |
| "input_tokens": 1024, |
| "output_tokens": 512, |
| "decode_tok_s": 43.15867903229062, |
| "ttft_s": 0.42960854701232165, |
| "e2e_aggregate_tok_s": 166.89079549066213, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "conc8-r0", |
| "batch": 8, |
| "input_tokens": 1024, |
| "output_tokens": 512, |
| "decode_tok_s": 41.003730058294124, |
| "ttft_s": 0.6135099662533321, |
| "e2e_aggregate_tok_s": 313.09421444193737, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "conc16-r0", |
| "batch": 16, |
| "input_tokens": 1024, |
| "output_tokens": 512, |
| "decode_tok_s": 37.32942669118185, |
| "ttft_s": 1.0930583830631804, |
| "e2e_aggregate_tok_s": 553.2943021614085, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "conc32-r0", |
| "batch": 32, |
| "input_tokens": 1024, |
| "output_tokens": 512, |
| "decode_tok_s": 31.83326072840652, |
| "ttft_s": 2.053213259844597, |
| "e2e_aggregate_tok_s": 900.9541856481276, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "conc64-r0", |
| "batch": 64, |
| "input_tokens": 1024, |
| "output_tokens": 512, |
| "decode_tok_s": 24.58912925154941, |
| "ttft_s": 3.9788458405942038, |
| "e2e_aggregate_tok_s": 1310.4877787340333, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-auto-bf16kv", |
| "case": "conc128-r0", |
| "batch": 128, |
| "input_tokens": 1024, |
| "output_tokens": 512, |
| "decode_tok_s": 15.826250676473936, |
| "ttft_s": 7.882490536335126, |
| "e2e_aggregate_tok_s": 1606.2952383754061, |
| "pool_tokens": 248865, |
| "kv_actual": "torch.bfloat16" |
| }, |
| { |
| "mode": "clean-unit-fp8kv", |
| "case": "single-r0", |
| "batch": 1, |
| "input_tokens": 1024, |
| "output_tokens": 1024, |
| "decode_tok_s": 44.07974218363006, |
| "ttft_s": 0.11572113499278203, |
| "e2e_aggregate_tok_s": 43.90329604550901, |
| "pool_tokens": 278528, |
| "kv_actual": "torch.float8_e4m3fn" |
| }, |
| { |
| "mode": "clean-unit-fp8kv", |
| "case": "long128k-r0", |
| "batch": 1, |
| "input_tokens": 131072, |
| "output_tokens": 256, |
| "decode_tok_s": 31.261125635351885, |
| "ttft_s": 58.35265536597581, |
| "e2e_aggregate_tok_s": 3.8487662647770065, |
| "pool_tokens": 278528, |
| "kv_actual": "torch.float8_e4m3fn" |
| } |
| ], |
| "receipt": "dad8f0cfdc609b357ffdc8d274f3c5ccb89a17ab6b8ba507242e6da1aca19cb9" |
| }, |
| "primitive_fp8_comparison": { |
| "revision": "7aab1aa", |
| "kv": "fp8_e4m3", |
| "single_1k_decode_tok_s": 33.42, |
| "decode_128k_tok_s": 25.41, |
| "aggregate_128_tok_s_initial_baseline": 1811, |
| "aggregate_128_tok_s_later_matched": 1854.5, |
| "aggregate_scope": "1811 is the initial8192-chunk baseline; later2048-chunk matched bank measured1854.5. These are not interchangeable exact-protocol concurrency controls.", |
| "primitive_actual_fp8_pool_tokens": 497671 |
| }, |
| "parser_contract": { |
| "recommended_reasoning_parser": "nanbeige", |
| "recommended_tool_call_parser": "nanbeige", |
| "n": 52, |
| "passed": 44, |
| "pins": { |
| "versions": { |
| "sglang": "0.5.21", |
| "torch": "2.13.0+cu130", |
| "transformers": "5.12.1", |
| "flashinfer-python": "0.6.18", |
| "sglang-kernel": "0.4.7" |
| }, |
| "artifact": "fp8-block-calib-v1", |
| "config_sha256": "e33990cbaf0e1cb7e261dee24c373e62fc8c1daa384e1a4787890adb5c9ea127", |
| "manifest_sha256": "5f3d0a025474f103d963541841d934f619b917f47d12fd547a3ed0f5d474ae98", |
| "script_sha256": "11bbda662b642c9efeca87e71941728d54da0ff595d995e776246a89fc0ae59c", |
| "aliases": { |
| "reasoning_same_class": true, |
| "tool_same_class": true |
| }, |
| "methods": "Native OpenAI endpoint, sampled output IDs via sglext decoded with skip_special_tokens=False. Natural cases plus separately labeled stock-regex wire fixtures. No parser or kernel patches." |
| }, |
| "known_limitations": [ |
| "Literal </parameter> inside an XML string parameter terminates it early in stock Qwen3CoderDetector, affecting both naming aliases and stream/non-stream." |
| ] |
| } |
| } |
|
|
