{ "status": "ESTIMATED screening only; not a qualified recalibration", "measured_worker_source": "offload/results/n2-pair-w5/summary.json: k2-tp2 draft_p50_us B1..4 k5_n5", "measured_worker_ms": [ 2.0275, 2.5122, 2.8961, 3.7219 ], "estimated_assumptions": { "nominal_anchor_caps": [ 5, 5, 4, 4 ], "central_overlap_fraction": 0.75, "overlap_sensitivity": [ 0, 1 ], "central_fixed_overhead_ms": 0.3, "overhead_sensitivity_ms": [ 0, 0.6 ], "central_row_slope_scale": 0.5, "row_slope_scale_sensitivity": [ 0, 1 ] }, "cells": [ { "suite": "W4-E1p", "context": 0, "batch": 1, "runs": 3, "request_rounds": 2308.0, "prefix_survival_MEASURED": [ 0.5558925476603119, 0.2785961871750433, 0.07279029462738301, 0.026429809358752165, 0.015164644714038129 ], "step_p50_median_MEASURED": 37.142, "step_p50_min_MEASURED": 36.935, "step_p50_max_MEASURED": 37.417, "per_run_survival_MEASURED": [ [ 0.5606860158311345, 0.2928759894459103, 0.08179419525065963, 0.027704485488126648, 0.014511873350923483 ], [ 0.5670650730411687, 0.28950863213811423, 0.0849933598937583, 0.03054448871181939, 0.0199203187250996 ], [ 0.5407779171894604, 0.2547051442910916, 0.05269761606022585, 0.02132998745294856, 0.011292346298619825 ] ], "sources_MEASURED": [ { "file": "W4-E1p-short.jsonl", "sha256": "598766fecc58668cf8de4b9a583e4a013230f4510f77e60f3dd249d14984f020" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.5558925476603118, "aggregate_tps_ESTIMATED": 49.817256264738454, "sensitivity_min_ESTIMATED": 39.2678689698844, "sensitivity_max_ESTIMATED": 64.46671938050213, "vs_nominal_cap_gain_percent_ESTIMATED": -5.057329384553444, "gain_sensitivity_min_ESTIMATED": -21.33333333333335, "gain_sensitivity_max_ESTIMATED": 21.756090807081986 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.8344887348353551, "aggregate_tps_ESTIMATED": 56.407623603571594, "sensitivity_min_ESTIMATED": 45.759218644873435, "sensitivity_max_ESTIMATED": 69.0480911249588, "vs_nominal_cap_gain_percent_ESTIMATED": 7.5027175228977505, "gain_sensitivity_min_ESTIMATED": -6.799999999999995, "gain_sensitivity_max_ESTIMATED": 28.269353352247627 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9072790294627382, "aggregate_tps_ESTIMATED": 55.797759916410335, "sensitivity_min_ESTIMATED": 47.10225650866427, "sensitivity_max_ESTIMATED": 64.27300817488808, "vs_nominal_cap_gain_percent_ESTIMATED": 6.3404277560191735, "gain_sensitivity_min_ESTIMATED": -2.533333333333332, "gain_sensitivity_max_ESTIMATED": 17.522972404295434 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9337088388214905, "aggregate_tps_ESTIMATED": 53.592063600174335, "sensitivity_min_ESTIMATED": 47.645867072579605, "sensitivity_max_ESTIMATED": 57.98942322709236, "vs_nominal_cap_gain_percent_ESTIMATED": 2.136769936784244, "gain_sensitivity_min_ESTIMATED": -1.0000000000000009, "gain_sensitivity_max_ESTIMATED": 5.595976376510192 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9488734835355286, "aggregate_tps_ESTIMATED": 52.47088157706985, "sensitivity_min_ESTIMATED": 47.93366090053479, "sensitivity_max_ESTIMATED": 55.13798107457721, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W4-E1p", "context": 32000, "batch": 1, "runs": 3, "request_rounds": 1770.0, "prefix_survival_MEASURED": [ 0.7084745762711865, 0.4598870056497175, 0.2271186440677966, 0.09096045197740113, 0.05706214689265537 ], "step_p50_median_MEASURED": 39.576, "step_p50_min_MEASURED": 39.463, "step_p50_max_MEASURED": 40.175, "per_run_survival_MEASURED": [ [ 0.7048903878583473, 0.45531197301854975, 0.2209106239460371, 0.0893760539629005, 0.06070826306913996 ], [ 0.6852459016393443, 0.44098360655737706, 0.19836065573770492, 0.08032786885245902, 0.05245901639344262 ], [ 0.7372134038800705, 0.48500881834215165, 0.26455026455026454, 0.10405643738977072, 0.0582010582010582 ] ], "sources_MEASURED": [ { "file": "W4-E1p-c32.jsonl", "sha256": "5aceafb928f29df9884a863eb7c536690a80e54ca43e1fae3ee62a64d3b1e3d4" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.7084745762711866, "aggregate_tps_ESTIMATED": 50.747774498639174, "sensitivity_min_ESTIMATED": 40.129082532748214, "sensitivity_max_ESTIMATED": 64.73413743154313, "vs_nominal_cap_gain_percent_ESTIMATED": -21.038266443849064, "gain_sensitivity_min_ESTIMATED": -34.420772303595214, "gain_sensitivity_max_ESTIMATED": -1.215230081749341 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.168361581920904, "aggregate_tps_ESTIMATED": 62.031170097291, "sensitivity_min_ESTIMATED": 50.629786035967356, "sensitivity_max_ESTIMATED": 75.54435610476303, "vs_nominal_cap_gain_percent_ESTIMATED": -3.4817039015195728, "gain_sensitivity_min_ESTIMATED": -16.111850865512658, "gain_sensitivity_max_ESTIMATED": 13.7028100193886 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.3954802259887007, "aggregate_tps_ESTIMATED": 65.42167975717447, "sensitivity_min_ESTIMATED": 55.35315080879083, "sensitivity_max_ESTIMATED": 75.96416762132007, "vs_nominal_cap_gain_percent_ESTIMATED": 1.7938086313591617, "gain_sensitivity_min_ESTIMATED": -6.125166444740371, "gain_sensitivity_max_ESTIMATED": 11.703207698327422 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.486440677966102, "aggregate_tps_ESTIMATED": 64.55604626560655, "sensitivity_min_ESTIMATED": 57.26591836142184, "sensitivity_max_ESTIMATED": 70.9114314712427, "vs_nominal_cap_gain_percent_ESTIMATED": 0.44691368288609645, "gain_sensitivity_min_ESTIMATED": -2.398401065956035, "gain_sensitivity_max_ESTIMATED": 3.543851071338766 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.543502824858757, "aggregate_tps_ESTIMATED": 64.26882011468457, "sensitivity_min_ESTIMATED": 58.51507268150738, "sensitivity_max_ESTIMATED": 68.52807885848144, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W4-E1p", "context": 32000, "batch": 3, "runs": 1, "request_rounds": 1680.0, "prefix_survival_MEASURED": [ 0.743452380952381, 0.5136904761904761, 0.2833333333333333, 0.13690476190476192, 0.0011904761904761906 ], "step_p50_median_MEASURED": 72.97, "step_p50_min_MEASURED": 72.97, "step_p50_max_MEASURED": 72.97, "per_run_survival_MEASURED": [ [ 0.743452380952381, 0.5136904761904761, 0.2833333333333333, 0.13690476190476192, 0.0011904761904761906 ] ], "sources_MEASURED": [ { "file": "W4-E1p-c32x3.jsonl", "sha256": "6e38b6adf06c7a4f0ab0436adc358c0a8d7f9f0757dd642b1bd55a8b2bc631fc" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.743452380952381, "aggregate_tps_ESTIMATED": 89.53024893627428, "sensitivity_min_ESTIMATED": 69.32944438308654, "sensitivity_max_ESTIMATED": 122.07347698020978, "vs_nominal_cap_gain_percent_ESTIMATED": -18.664018045069742, "gain_sensitivity_min_ESTIMATED": -34.88216985326812, "gain_sensitivity_max_ESTIMATED": 9.344361513328137 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.257142857142857, "aggregate_tps_ESTIMATED": 108.70811641400823, "sensitivity_min_ESTIMATED": 89.7566586209164, "sensitivity_max_ESTIMATED": 133.8598014850672, "vs_nominal_cap_gain_percent_ESTIMATED": -1.2414072332392, "gain_sensitivity_min_ESTIMATED": -15.6958648288128, "gain_sensitivity_max_ESTIMATED": 19.90167633267632 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.5404761904761903, "aggregate_tps_ESTIMATED": 113.29609887659538, "sensitivity_min_ESTIMATED": 101.02358095835208, "sensitivity_max_ESTIMATED": 125.87836881689611, "vs_nominal_cap_gain_percent_ESTIMATED": 2.9266595734569245, "gain_sensitivity_min_ESTIMATED": -5.113383726100507, "gain_sensitivity_max_ESTIMATED": 12.752501256715544 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.677380952380952, "aggregate_tps_ESTIMATED": 110.07459034045303, "sensitivity_min_ESTIMATED": 106.46768208778532, "sensitivity_max_ESTIMATED": 111.64130942895494, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.678571428571429, "aggregate_tps_ESTIMATED": 105.52481005534192, "sensitivity_min_ESTIMATED": 98.23362409467349, "sensitivity_max_ESTIMATED": 111.690949851111, "vs_nominal_cap_gain_percent_ESTIMATED": -4.133361088184806, "gain_sensitivity_min_ESTIMATED": -8.081132752949472, "gain_sensitivity_max_ESTIMATED": 0.04446420631392112 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W4-E1p", "context": 150000, "batch": 1, "runs": 2, "request_rounds": 1367.0, "prefix_survival_MEASURED": [ 0.6569129480614484, 0.36430138990490124, 0.11485003657644477, 0.03511338697878566, 0.021945866861741038 ], "step_p50_median_MEASURED": 39.697, "step_p50_min_MEASURED": 39.599, "step_p50_max_MEASURED": 39.795, "per_run_survival_MEASURED": [ [ 0.6588921282798834, 0.36588921282798836, 0.11078717201166181, 0.029154518950437316, 0.02040816326530612 ], [ 0.6549192364170338, 0.36270190895741555, 0.11894273127753303, 0.041116005873715125, 0.023494860499265784 ] ], "sources_MEASURED": [ { "file": "W4-E1p-c150.jsonl", "sha256": "f4312374bd081d33c64171c0ebb96d56ad195244d50fa6284bf600b29fe6373c" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.6569129480614484, "aggregate_tps_ESTIMATED": 49.03995465893534, "sensitivity_min_ESTIMATED": 39.76677597457767, "sensitivity_max_ESTIMATED": 61.50394632532229, "vs_nominal_cap_gain_percent_ESTIMATED": -11.234412191764543, "gain_sensitivity_min_ESTIMATED": -24.816544362908623, "gain_sensitivity_max_ESTIMATED": 9.186522939146258 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.0212143379663496, "aggregate_tps_ESTIMATED": 57.62221221787352, "sensitivity_min_ESTIMATED": 48.48229830440969, "sensitivity_max_ESTIMATED": 68.51559206344288, "vs_nominal_cap_gain_percent_ESTIMATED": 4.300046235839505, "gain_sensitivity_min_ESTIMATED": -8.338892595063385, "gain_sensitivity_max_ESTIMATED": 21.63413425463787 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1360643745427943, "aggregate_tps_ESTIMATED": 58.14476888539603, "sensitivity_min_ESTIMATED": 51.31650703598789, "sensitivity_max_ESTIMATED": 64.99622025202152, "vs_nominal_cap_gain_percent_ESTIMATED": 5.245908646976494, "gain_sensitivity_min_ESTIMATED": -2.9352901934623254, "gain_sensitivity_max_ESTIMATED": 15.33252033498529 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.17117776152158, "aggregate_tps_ESTIMATED": 56.19426356915857, "sensitivity_min_ESTIMATED": 52.017073684943355, "sensitivity_max_ESTIMATED": 59.38242963901594, "vs_nominal_cap_gain_percent_ESTIMATED": 1.7153639692121914, "gain_sensitivity_min_ESTIMATED": -1.0673782521681274, "gain_sensitivity_max_ESTIMATED": 4.793010038064649 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1931236283833213, "aggregate_tps_ESTIMATED": 55.246583580203065, "sensitivity_min_ESTIMATED": 52.50747033921218, "sensitivity_max_ESTIMATED": 56.74282455588508, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W5-E1p", "context": 0, "batch": 1, "runs": 3, "request_rounds": 2267.0, "prefix_survival_MEASURED": [ 0.5756506396118218, 0.29422143802382, 0.07587119541243935, 0.026025584472871635, 0.012792236435818262 ], "step_p50_median_MEASURED": 37.255, "step_p50_min_MEASURED": 37.194, "step_p50_max_MEASURED": 37.286, "per_run_survival_MEASURED": [ [ 0.5844327176781002, 0.28891820580474936, 0.07255936675461741, 0.021108179419525065, 0.010554089709762533 ], [ 0.5850340136054422, 0.31700680272108844, 0.09115646258503401, 0.031292517006802724, 0.017687074829931974 ], [ 0.5581395348837209, 0.2777777777777778, 0.06459948320413436, 0.025839793281653745, 0.0103359173126615 ] ], "sources_MEASURED": [ { "file": "W5-E1p-short.jsonl", "sha256": "dfc633b621a1924a4534e410147b2cbeb709591de852b2cf5e30a07cba0b7155" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.5756506396118217, "aggregate_tps_ESTIMATED": 50.268005730158606, "sensitivity_min_ESTIMATED": 39.84336502788776, "sensitivity_max_ESTIMATED": 64.51849834302719, "vs_nominal_cap_gain_percent_ESTIMATED": -5.6348247892310965, "gain_sensitivity_min_ESTIMATED": -22.385076615589604, "gain_sensitivity_max_ESTIMATED": 19.162356238292745 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.869872077635642, "aggregate_tps_ESTIMATED": 57.2965245177154, "sensitivity_min_ESTIMATED": 46.94645249140008, "sensitivity_max_ESTIMATED": 70.06417130088474, "vs_nominal_cap_gain_percent_ESTIMATED": 7.559400786780923, "gain_sensitivity_min_ESTIMATED": -6.862091938707538, "gain_sensitivity_max_ESTIMATED": 27.06219254583595 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9457432730480813, "aggregate_tps_ESTIMATED": 56.735479604842716, "sensitivity_min_ESTIMATED": 48.59833329686806, "sensitivity_max_ESTIMATED": 65.42124597944716, "vs_nominal_cap_gain_percent_ESTIMATED": 6.50618411873678, "gain_sensitivity_min_ESTIMATED": -2.398401065956035, "gain_sensitivity_max_ESTIMATED": 17.518612641465726 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9717688575209529, "aggregate_tps_ESTIMATED": 54.47627731788791, "sensitivity_min_ESTIMATED": 49.259085619055256, "sensitivity_max_ESTIMATED": 59.07965129605613, "vs_nominal_cap_gain_percent_ESTIMATED": 2.265116335195194, "gain_sensitivity_min_ESTIMATED": -0.8660892738174386, "gain_sensitivity_max_ESTIMATED": 5.619980872572139 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.984561093956771, "aggregate_tps_ESTIMATED": 53.26965760184595, "sensitivity_min_ESTIMATED": 49.52338654793014, "sensitivity_max_ESTIMATED": 56.12361160021021, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W5-E1p", "context": 32000, "batch": 1, "runs": 3, "request_rounds": 1726.0, "prefix_survival_MEASURED": [ 0.7149478563151797, 0.47334878331402086, 0.25028968713789107, 0.10081112398609501, 0.06720741599073002 ], "step_p50_median_MEASURED": 40.384, "step_p50_min_MEASURED": 40.352, "step_p50_max_MEASURED": 40.484, "per_run_survival_MEASURED": [ [ 0.727112676056338, 0.4894366197183099, 0.2482394366197183, 0.10915492957746478, 0.06514084507042253 ], [ 0.6961602671118531, 0.44073455759599334, 0.2287145242070117, 0.08180300500834725, 0.05509181969949917 ], [ 0.7227191413237924, 0.4919499105545617, 0.27549194991055453, 0.11270125223613596, 0.08228980322003578 ] ], "sources_MEASURED": [ { "file": "W5-E1p-c32.jsonl", "sha256": "f550e2cc5115c0daf4dcc4e20c2a3119c46c7e300a4cb82fa7db5a2457fb0906" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.7149478563151797, "aggregate_tps_ESTIMATED": 49.746123348470725, "sensitivity_min_ESTIMATED": 40.09396767166362, "sensitivity_max_ESTIMATED": 62.294134870675535, "vs_nominal_cap_gain_percent_ESTIMATED": -22.928577670686558, "gain_sensitivity_min_ESTIMATED": -35.84277148567623, "gain_sensitivity_max_ESTIMATED": -3.3255934221716354 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1882966396292005, "aggregate_tps_ESTIMATED": 61.18713341989711, "sensitivity_min_ESTIMATED": 50.512085255639214, "sensitivity_max_ESTIMATED": 73.14107088403853, "vs_nominal_cap_gain_percent_ESTIMATED": -5.203077476188723, "gain_sensitivity_min_ESTIMATED": -17.52165223184544, "gain_sensitivity_max_ESTIMATED": 11.425658608620214 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.4385863267670915, "aggregate_tps_ESTIMATED": 65.16102839800908, "sensitivity_min_ESTIMATED": 55.91845688065684, "sensitivity_max_ESTIMATED": 74.05655746376881, "vs_nominal_cap_gain_percent_ESTIMATED": 0.9536583161656509, "gain_sensitivity_min_ESTIMATED": -7.261825449700199, "gain_sensitivity_max_ESTIMATED": 11.172471988672704 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.539397450753187, "aggregate_tps_ESTIMATED": 64.57627532176754, "sensitivity_min_ESTIMATED": 57.852122644349286, "sensitivity_max_ESTIMATED": 69.5485253295759, "vs_nominal_cap_gain_percent_ESTIMATED": 0.04770327356509174, "gain_sensitivity_min_ESTIMATED": -3.0646235842771707, "gain_sensitivity_max_ESTIMATED": 3.3384707944514203 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.6066048667439166, "aggregate_tps_ESTIMATED": 64.54548501247812, "sensitivity_min_ESTIMATED": 59.15438734234624, "sensitivity_max_ESTIMATED": 67.90096269123136, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W5-E1p", "context": 32000, "batch": 3, "runs": 1, "request_rounds": 1703.0, "prefix_survival_MEASURED": [ 0.7322372284204345, 0.4938344098649442, 0.278919553728714, 0.13329418672930124, 0.002935995302407516 ], "step_p50_median_MEASURED": 73.33, "step_p50_min_MEASURED": 73.33, "step_p50_max_MEASURED": 73.33, "per_run_survival_MEASURED": [ [ 0.7322372284204345, 0.4938344098649442, 0.278919553728714, 0.13329418672930124, 0.002935995302407516 ] ], "sources_MEASURED": [ { "file": "W5-E1p-c32x3.jsonl", "sha256": "0eeacd0d663465af1288ef9b70771f46be44021871ddf3fe7d507dc298dd4d7a" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.7322372284204346, "aggregate_tps_ESTIMATED": 88.40952169549684, "sensitivity_min_ESTIMATED": 68.55632494573932, "sensitivity_max_ESTIMATED": 120.27761635424972, "vs_nominal_cap_gain_percent_ESTIMATED": -18.089853885598906, "gain_sensitivity_min_ESTIMATED": -34.34231026040507, "gain_sensitivity_max_ESTIMATED": 9.879322775817645 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.2260716382853785, "aggregate_tps_ESTIMATED": 106.59560917567656, "sensitivity_min_ESTIMATED": 88.10068741332128, "sensitivity_max_ESTIMATED": 131.0842498324183, "vs_nominal_cap_gain_percent_ESTIMATED": -1.2407062577967753, "gain_sensitivity_min_ESTIMATED": -15.62430447362565, "gain_sensitivity_max_ESTIMATED": 19.751696406588295 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.5049911920140926, "aggregate_tps_ESTIMATED": 111.1189350294585, "sensitivity_min_ESTIMATED": 99.13941770119456, "sensitivity_max_ESTIMATED": 123.38647523567728, "vs_nominal_cap_gain_percent_ESTIMATED": 2.9500898748012094, "gain_sensitivity_min_ESTIMATED": -5.0523035833518914, "gain_sensitivity_max_ESTIMATED": 12.719413217007848 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.6382853787433937, "aggregate_tps_ESTIMATED": 107.93476252870833, "sensitivity_min_ESTIMATED": 104.41476880718874, "sensitivity_max_ESTIMATED": 109.46337610730208, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.6412213740458013, "aggregate_tps_ESTIMATED": 103.56377103826173, "sensitivity_min_ESTIMATED": 96.43943537377073, "sensitivity_max_ESTIMATED": 109.58519157147668, "vs_nominal_cap_gain_percent_ESTIMATED": -4.049660543130407, "gain_sensitivity_min_ESTIMATED": -7.982529752897472, "gain_sensitivity_max_ESTIMATED": 0.11128421989763382 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W5-E1p", "context": 32000, "batch": 4, "runs": 1, "request_rounds": 2240.0, "prefix_survival_MEASURED": [ 0.728125, 0.5013392857142858, 0.2892857142857143, 0.1575892857142857, 0.0013392857142857143 ], "step_p50_median_MEASURED": 87.667, "step_p50_min_MEASURED": 87.667, "step_p50_max_MEASURED": 87.667, "per_run_survival_MEASURED": [ [ 0.728125, 0.5013392857142858, 0.2892857142857143, 0.1575892857142857, 0.0013392857142857143 ] ], "sources_MEASURED": [ { "file": "W5-E1p-c32x4.jsonl", "sha256": "603ece0fe0114704b729643f1aaa3c1f05effbe69aa209f17172d8082e82a365" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.728125, "aggregate_tps_ESTIMATED": 101.25682980063574, "sensitivity_min_ESTIMATED": 76.16372805059144, "sensitivity_max_ESTIMATED": 145.10924128071898, "vs_nominal_cap_gain_percent_ESTIMATED": -17.079996681332666, "gain_sensitivity_min_ESTIMATED": -35.42952460383654, "gain_sensitivity_max_ESTIMATED": 17.16319590571247 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.2294642857142857, "aggregate_tps_ESTIMATED": 121.45201551005957, "sensitivity_min_ESTIMATED": 98.25927612623448, "sensitivity_max_ESTIMATED": 153.87149493274737, "vs_nominal_cap_gain_percent_ESTIMATED": -0.5420024606602292, "gain_sensitivity_min_ESTIMATED": -16.697247706422026, "gain_sensitivity_max_ESTIMATED": 24.237959939673104 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.51875, "aggregate_tps_ESTIMATED": 125.83211560318234, "sensitivity_min_ESTIMATED": 111.00897795438826, "sensitivity_max_ESTIMATED": 141.43025645902858, "vs_nominal_cap_gain_percent_ESTIMATED": 3.044895479685472, "gain_sensitivity_min_ESTIMATED": -5.888240200166816, "gain_sensitivity_max_ESTIMATED": 14.192732993815715 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.6763392857142856, "aggregate_tps_ESTIMATED": 122.11387572127646, "sensitivity_min_ESTIMATED": 117.95441737620659, "sensitivity_max_ESTIMATED": 123.8522388869421, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.6776785714285714, "aggregate_tps_ESTIMATED": 116.53861279025848, "sensitivity_min_ESTIMATED": 107.92910393040081, "sensitivity_max_ESTIMATED": 123.91421665452522, "vs_nominal_cap_gain_percent_ESTIMATED": -4.565626058535277, "gain_sensitivity_min_ESTIMATED": -8.888595207466278, "gain_sensitivity_max_ESTIMATED": 0.050041701417846696 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W5-E1p", "context": 150000, "batch": 1, "runs": 2, "request_rounds": 1356.0, "prefix_survival_MEASURED": [ 0.6349557522123894, 0.3812684365781711, 0.13421828908554573, 0.04056047197640118, 0.021386430678466076 ], "step_p50_median_MEASURED": 39.672, "step_p50_min_MEASURED": 39.395, "step_p50_max_MEASURED": 39.949, "per_run_survival_MEASURED": [ [ 0.6515625, 0.425, 0.1734375, 0.065625, 0.0296875 ], [ 0.6201117318435754, 0.34217877094972066, 0.09916201117318436, 0.018156424581005588, 0.013966480446927373 ] ], "sources_MEASURED": [ { "file": "W5-E1p-c150.jsonl", "sha256": "f052aa7a7c7c60487c0404d572784d8a9ed146f44d14c911abd9d4165e17cd44" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.6349557522123894, "aggregate_tps_ESTIMATED": 48.425915295669384, "sensitivity_min_ESTIMATED": 38.786839284374125, "sensitivity_max_ESTIMATED": 61.69884891078474, "vs_nominal_cap_gain_percent_ESTIMATED": -13.163888395237267, "gain_sensitivity_min_ESTIMATED": -29.58027981345771, "gain_sensitivity_max_ESTIMATED": 11.555781091446837 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.0162241887905603, "aggregate_tps_ESTIMATED": 57.520945703256885, "sensitivity_min_ESTIMATED": 46.978887236677274, "sensitivity_max_ESTIMATED": 70.75622377920224, "vs_nominal_cap_gain_percent_ESTIMATED": 3.1450872988701972, "gain_sensitivity_min_ESTIMATED": -11.459027315123272, "gain_sensitivity_max_ESTIMATED": 23.23895523736015 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.150442477876106, "aggregate_tps_ESTIMATED": 58.576009966117525, "sensitivity_min_ESTIMATED": 49.35290929632431, "sensitivity_max_ESTIMATED": 68.87447626700337, "vs_nominal_cap_gain_percent_ESTIMATED": 5.037001525386775, "gain_sensitivity_min_ESTIMATED": -4.0639573617588525, "gain_sensitivity_max_ESTIMATED": 16.30923322698017 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1910029498525074, "aggregate_tps_ESTIMATED": 56.74409380121484, "sensitivity_min_ESTIMATED": 49.78758939175265, "sensitivity_max_ESTIMATED": 63.49723217193095, "vs_nominal_cap_gain_percent_ESTIMATED": 1.7520563555371016, "gain_sensitivity_min_ESTIMATED": -1.2658227848101555, "gain_sensitivity_max_ESTIMATED": 5.107403290254431 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.212389380530974, "aggregate_tps_ESTIMATED": 55.76702411098443, "sensitivity_min_ESTIMATED": 50.12195869592828, "sensitivity_max_ESTIMATED": 60.778089114202885, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W5-prod", "context": 0, "batch": 1, "runs": 2, "request_rounds": 1503.0, "prefix_survival_MEASURED": [ 0.5788423153692615, 0.30073186959414505, 0.08183632734530938, 0.02262142381902861, 0.011310711909514305 ], "step_p50_median_MEASURED": 38.4765, "step_p50_min_MEASURED": 38.44, "step_p50_max_MEASURED": 38.513, "per_run_survival_MEASURED": [ [ 0.5816993464052288, 0.27712418300653596, 0.06928104575163399, 0.022222222222222223, 0.01045751633986928 ], [ 0.575880758807588, 0.3252032520325203, 0.0948509485094851, 0.023035230352303523, 0.012195121951219513 ] ], "sources_MEASURED": [ { "file": "W5-prod-short.jsonl", "sha256": "d9b6b56efc230441a07ee97131fe6ea225b4e60bae37f58f7399be93921de8ac" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.5788423153692615, "aggregate_tps_ESTIMATED": 48.480564855580475, "sensitivity_min_ESTIMATED": 39.071141232844504, "sensitivity_max_ESTIMATED": 61.27500434004906, "vs_nominal_cap_gain_percent_ESTIMATED": -6.514179131056663, "gain_sensitivity_min_ESTIMATED": -22.414943295530353, "gain_sensitivity_max_ESTIMATED": 17.604464782935025 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.8795741849634064, "aggregate_tps_ESTIMATED": 55.515903444343216, "sensitivity_min_ESTIMATED": 46.08620051908959, "sensitivity_max_ESTIMATED": 66.95578633349125, "vs_nominal_cap_gain_percent_ESTIMATED": 7.05217276729031, "gain_sensitivity_min_ESTIMATED": -6.404269513008664, "gain_sensitivity_max_ESTIMATED": 25.65084857690023 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9614105123087158, "aggregate_tps_ESTIMATED": 55.22533223455903, "sensitivity_min_ESTIMATED": 47.8038999758489, "sensitivity_max_ESTIMATED": 62.93718955005517, "vs_nominal_cap_gain_percent_ESTIMATED": 6.491859822330293, "gain_sensitivity_min_ESTIMATED": -1.7344896597731863, "gain_sensitivity_max_ESTIMATED": 16.689560710273742 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9840319361277445, "aggregate_tps_ESTIMATED": 53.02558860737226, "sensitivity_min_ESTIMATED": 48.35486017896038, "sensitivity_max_ESTIMATED": 56.85138071352203, "vs_nominal_cap_gain_percent_ESTIMATED": 2.2500602620037835, "gain_sensitivity_min_ESTIMATED": -0.6004002668445585, "gain_sensitivity_max_ESTIMATED": 5.404452579152075 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.995342648037259, "aggregate_tps_ESTIMATED": 51.85873580074224, "sensitivity_min_ESTIMATED": 48.6141355686599, "sensitivity_max_ESTIMATED": 53.972804853519776, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "W5-prod", "context": 32000, "batch": 1, "runs": 1, "request_rounds": 617.0, "prefix_survival_MEASURED": [ 0.6871961102106969, 0.4343598055105348, 0.21231766612641814, 0.06482982171799027, 0.03079416531604538 ], "step_p50_median_MEASURED": 40.457, "step_p50_min_MEASURED": 40.457, "step_p50_max_MEASURED": 40.457, "per_run_survival_MEASURED": [ [ 0.6871961102106969, 0.4343598055105348, 0.21231766612641814, 0.06482982171799027, 0.03079416531604538 ] ], "sources_MEASURED": [ { "file": "W5-prod-c32.jsonl", "sha256": "2cc57b9ded76ef3661be881c8a099dd4a83bd7e184b0ff806d1659ffe4b393bd" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.687196110210697, "aggregate_tps_ESTIMATED": 48.83770255624792, "sensitivity_min_ESTIMATED": 39.90754235155586, "sensitivity_max_ESTIMATED": 60.62481250841299, "vs_nominal_cap_gain_percent_ESTIMATED": -18.673434473630323, "gain_sensitivity_min_ESTIMATED": -30.553702468312206, "gain_sensitivity_max_ESTIMATED": -1.0584976560970127 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.121555915721232, "aggregate_tps_ESTIMATED": 59.20015391135509, "sensitivity_min_ESTIMATED": 50.181530199987144, "sensitivity_max_ESTIMATED": 69.7647877383349, "vs_nominal_cap_gain_percent_ESTIMATED": -1.4174511854253002, "gain_sensitivity_min_ESTIMATED": -12.675116744496318, "gain_sensitivity_max_ESTIMATED": 13.85821454831666 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.3338735818476497, "aggregate_tps_ESTIMATED": 62.241608177924896, "sensitivity_min_ESTIMATED": 55.20351679754124, "sensitivity_max_ESTIMATED": 69.19255656027512, "vs_nominal_cap_gain_percent_ESTIMATED": 3.6473044594735216, "gain_sensitivity_min_ESTIMATED": -3.935957304869908, "gain_sensitivity_max_ESTIMATED": 12.924316197086295 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.3987034035656403, "aggregate_tps_ESTIMATED": 60.88543299148768, "sensitivity_min_ESTIMATED": 56.73694781969517, "sensitivity_max_ESTIMATED": 63.914079784323654, "vs_nominal_cap_gain_percent_ESTIMATED": 1.3889453559101428, "gain_sensitivity_min_ESTIMATED": -1.2675116744496284, "gain_sensitivity_max_ESTIMATED": 4.309684651173229 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.4294975688816853, "aggregate_tps_ESTIMATED": 60.051352519506764, "sensitivity_min_ESTIMATED": 57.46532755521827, "sensitivity_max_ESTIMATED": 61.27338990436184, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "K1-R1", "context": 0, "batch": 1, "runs": 3, "request_rounds": 2229.0, "prefix_survival_MEASURED": [ 0.5903992821893226, 0.30372364288918796, 0.08568864961866307, 0.02736653207716465, 0.011215791834903545 ], "step_p50_median_MEASURED": 38.628, "step_p50_min_MEASURED": 38.245, "step_p50_max_MEASURED": 39.311, "per_run_survival_MEASURED": [ [ 0.5699481865284974, 0.27461139896373055, 0.06994818652849741, 0.022020725388601035, 0.006476683937823834 ], [ 0.5823451910408433, 0.2845849802371542, 0.07378129117259552, 0.023715415019762844, 0.010540184453227932 ], [ 0.6217765042979942, 0.35673352435530087, 0.11604584527220631, 0.03724928366762178, 0.017191977077363897 ] ], "sources_MEASURED": [ { "file": "R1-short.jsonl", "sha256": "6cbc32a73e219b98eb2b51deda1288167c2d2d6ae07d8374524f2b9db739413f" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.5903992821893227, "aggregate_tps_ESTIMATED": 48.60930625922498, "sensitivity_min_ESTIMATED": 38.168883104630495, "sensitivity_max_ESTIMATED": 63.30582368139722, "vs_nominal_cap_gain_percent_ESTIMATED": -6.971563703440486, "gain_sensitivity_min_ESTIMATED": -24.53333333333334, "gain_sensitivity_max_ESTIMATED": 18.08048012881507 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.8941229250785105, "aggregate_tps_ESTIMATED": 55.696392762835515, "sensitivity_min_ESTIMATED": 44.845288400159916, "sensitivity_max_ESTIMATED": 70.16459529324362, "vs_nominal_cap_gain_percent_ESTIMATED": 6.591694570878515, "gain_sensitivity_min_ESTIMATED": -7.933333333333337, "gain_sensitivity_max_ESTIMATED": 26.041217279517713 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9798115746971736, "aggregate_tps_ESTIMATED": 55.50666072381893, "sensitivity_min_ESTIMATED": 46.54588220185138, "sensitivity_max_ESTIMATED": 66.45559892682391, "vs_nominal_cap_gain_percent_ESTIMATED": 6.228585605468795, "gain_sensitivity_min_ESTIMATED": -2.533333333333332, "gain_sensitivity_max_ESTIMATED": 17.040694838287518 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.0071781067743384, "aggregate_tps_ESTIMATED": 53.42786698185525, "sensitivity_min_ESTIMATED": 47.08125432460609, "sensitivity_max_ESTIMATED": 60.360088696473085, "vs_nominal_cap_gain_percent_ESTIMATED": 2.2501924523829375, "gain_sensitivity_min_ESTIMATED": -0.7999999999999896, "gain_sensitivity_max_ESTIMATED": 5.64924171370933 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.018393898609242, "aggregate_tps_ESTIMATED": 52.25209429971114, "sensitivity_min_ESTIMATED": 47.2387167136516, "sensitivity_max_ESTIMATED": 57.40130240685096, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "K1-R1", "context": 32000, "batch": 1, "runs": 2, "request_rounds": 1199.0, "prefix_survival_MEASURED": [ 0.7147623019182652, 0.457047539616347, 0.213511259382819, 0.07422852376980818, 0.043369474562135114 ], "step_p50_median_MEASURED": 40.9405, "step_p50_min_MEASURED": 40.9, "step_p50_max_MEASURED": 40.981, "per_run_survival_MEASURED": [ [ 0.7271186440677966, 0.4694915254237288, 0.21864406779661016, 0.08135593220338982, 0.04576271186440678 ], [ 0.7027914614121511, 0.44499178981937604, 0.20853858784893267, 0.06732348111658457, 0.041050903119868636 ] ], "sources_MEASURED": [ { "file": "R1-c32.jsonl", "sha256": "4935297483933f3ff5a87bde7c55be5a6825c24df06421cfd2c3593111224342" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.7147623019182652, "aggregate_tps_ESTIMATED": 48.95055171688287, "sensitivity_min_ESTIMATED": 39.783336763782934, "sensitivity_max_ESTIMATED": 61.08693835816864, "vs_nominal_cap_gain_percent_ESTIMATED": -19.931089136438295, "gain_sensitivity_min_ESTIMATED": -32.06666666666666, "gain_sensitivity_max_ESTIMATED": -2.029762946461089 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1718098415346123, "aggregate_tps_ESTIMATED": 59.7957032952358, "sensitivity_min_ESTIMATED": 50.17994646772235, "sensitivity_max_ESTIMATED": 71.19571095282976, "vs_nominal_cap_gain_percent_ESTIMATED": -2.1915653808876057, "gain_sensitivity_min_ESTIMATED": -13.59999999999999, "gain_sensitivity_max_ESTIMATED": 13.23946296513354 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.385321100917431, "aggregate_tps_ESTIMATED": 62.80383620324722, "sensitivity_min_ESTIMATED": 55.05215839539875, "sensitivity_max_ESTIMATED": 70.67700824224111, "vs_nominal_cap_gain_percent_ESTIMATED": 2.7288679386461956, "gain_sensitivity_min_ESTIMATED": -4.999999999999993, "gain_sensitivity_max_ESTIMATED": 12.164757485535826 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.4595496246872393, "aggregate_tps_ESTIMATED": 61.672988670835096, "sensitivity_min_ESTIMATED": 56.62507720669586, "sensitivity_max_ESTIMATED": 65.74676615347104, "vs_nominal_cap_gain_percent_ESTIMATED": 0.8791292309666909, "gain_sensitivity_min_ESTIMATED": -1.8000000000000127, "gain_sensitivity_max_ESTIMATED": 3.824353229760291 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.5029190992493744, "aggregate_tps_ESTIMATED": 61.13552837042475, "sensitivity_min_ESTIMATED": 57.58417404285264, "sensitivity_max_ESTIMATED": 63.411691689184416, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "K1-R1", "context": 32000, "batch": 2, "runs": 2, "request_rounds": 2241.0, "prefix_survival_MEASURED": [ 0.7202141900937081, 0.49040606871932174, 0.2788933511825078, 0.11601963409192324, 0.07095046854082998 ], "step_p50_median_MEASURED": 61.03875, "step_p50_min_MEASURED": 60.55, "step_p50_max_MEASURED": 61.5275, "per_run_survival_MEASURED": [ [ 0.7000871839581517, 0.4664341761115955, 0.26416739319965127, 0.11072362685265912, 0.07236268526591107 ], [ 0.7413162705667276, 0.5155393053016454, 0.2943327239488117, 0.12157221206581353, 0.06946983546617916 ] ], "sources_MEASURED": [ { "file": "R1-c32.jsonl", "sha256": "4935297483933f3ff5a87bde7c55be5a6825c24df06421cfd2c3593111224342" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.7202141900937082, "aggregate_tps_ESTIMATED": 69.90076708952212, "sensitivity_min_ESTIMATED": 53.36817313499655, "sensitivity_max_ESTIMATED": 96.7883213981859, "vs_nominal_cap_gain_percent_ESTIMATED": -20.293640737388642, "gain_sensitivity_min_ESTIMATED": -36.5, "gain_sensitivity_max_ESTIMATED": 7.776596082042864 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.21062025881303, "aggregate_tps_ESTIMATED": 85.35419325034022, "sensitivity_min_ESTIMATED": 68.01021037972637, "sensitivity_max_ESTIMATED": 109.71067612830083, "vs_nominal_cap_gain_percent_ESTIMATED": -2.6724272843958397, "gain_sensitivity_min_ESTIMATED": -17.700000000000003, "gain_sensitivity_max_ESTIMATED": 20.120206051023203 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.4895136099955377, "aggregate_tps_ESTIMATED": 90.33273105778116, "sensitivity_min_ESTIMATED": 76.30280343608737, "sensitivity_max_ESTIMATED": 106.7846038019455, "vs_nominal_cap_gain_percent_ESTIMATED": 3.0044935793504868, "gain_sensitivity_min_ESTIMATED": -7.004669779853268, "gain_sensitivity_max_ESTIMATED": 16.08627836104082 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.605533244087461, "aggregate_tps_ESTIMATED": 88.44496001994139, "sensitivity_min_ESTIMATED": 79.77857676334101, "sensitivity_max_ESTIMATED": 96.52099689097254, "vs_nominal_cap_gain_percent_ESTIMATED": 0.8519083816100226, "gain_sensitivity_min_ESTIMATED": -2.768512341561058, "gain_sensitivity_max_ESTIMATED": 4.928640588976507 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.6764837126282908, "aggregate_tps_ESTIMATED": 87.6978546457223, "sensitivity_min_ESTIMATED": 82.05014515831779, "sensitivity_max_ESTIMATED": 91.98727473184547, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "K1-R1", "context": 32000, "batch": 3, "runs": 2, "request_rounds": 3321.0, "prefix_survival_MEASURED": [ 0.7425474254742548, 0.5097862089732008, 0.29659741041854865, 0.15356820234869015, 0.006925624811803674 ], "step_p50_median_MEASURED": 75.20349999999999, "step_p50_min_MEASURED": 74.857, "step_p50_max_MEASURED": 75.55, "per_run_survival_MEASURED": [ [ 0.7425076452599388, 0.5168195718654435, 0.309480122324159, 0.1712538226299694, 0.011620795107033639 ], [ 0.7425860023724793, 0.5029655990510083, 0.28410438908659547, 0.13641755634638197, 0.002372479240806643 ] ], "sources_MEASURED": [ { "file": "R1-c32.jsonl", "sha256": "4935297483933f3ff5a87bde7c55be5a6825c24df06421cfd2c3593111224342" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.7425474254742548, "aggregate_tps_ESTIMATED": 86.18863340817539, "sensitivity_min_ESTIMATED": 67.00056279943614, "sensitivity_max_ESTIMATED": 116.86586924114567, "vs_nominal_cap_gain_percent_ESTIMATED": -20.05317671297525, "gain_sensitivity_min_ESTIMATED": -36.40625000000001, "gain_sensitivity_max_ESTIMATED": 7.8809298941834305 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.2523336344474556, "aggregate_tps_ESTIMATED": 104.72154956476894, "sensitivity_min_ESTIMATED": 86.34293312848273, "sensitivity_max_ESTIMATED": 129.17090466809148, "vs_nominal_cap_gain_percent_ESTIMATED": -2.8624206425362897, "gain_sensitivity_min_ESTIMATED": -17.544642857142854, "gain_sensitivity_max_ESTIMATED": 18.51289144533692 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.5489310448660043, "aggregate_tps_ESTIMATED": 110.02025990918465, "sensitivity_min_ESTIMATED": 97.26693338430499, "sensitivity_max_ESTIMATED": 123.43512412068502, "vs_nominal_cap_gain_percent_ESTIMATED": 2.0525552980608985, "gain_sensitivity_min_ESTIMATED": -6.25, "gain_sensitivity_max_ESTIMATED": 12.208479514699654 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.7024992472146945, "aggregate_tps_ESTIMATED": 107.80745233458661, "sensitivity_min_ESTIMATED": 102.51227797478336, "sensitivity_max_ESTIMATED": 111.33485392182193, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.7094248720264984, "aggregate_tps_ESTIMATED": 103.69879650793212, "sensitivity_min_ESTIMATED": 94.87012589216152, "sensitivity_max_ESTIMATED": 111.80703298979395, "vs_nominal_cap_gain_percent_ESTIMATED": -3.8111055754318834, "gain_sensitivity_min_ESTIMATED": -7.848938995364751, "gain_sensitivity_max_ESTIMATED": 0.42410714285716633 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "K1-R1", "context": 150000, "batch": 1, "runs": 2, "request_rounds": 1396.0, "prefix_survival_MEASURED": [ 0.6131805157593123, 0.34025787965616044, 0.1174785100286533, 0.042263610315186245, 0.034383954154727794 ], "step_p50_median_MEASURED": 40.474500000000006, "step_p50_min_MEASURED": 40.429, "step_p50_max_MEASURED": 40.52, "per_run_survival_MEASURED": [ [ 0.6149341142020498, 0.3557833089311859, 0.13323572474377746, 0.05124450951683748, 0.03953147877013177 ], [ 0.6115007012622721, 0.32538569424964936, 0.10238429172510519, 0.033660589060308554, 0.029453015427769985 ] ], "sources_MEASURED": [ { "file": "R1-c150.jsonl", "sha256": "2575f6a8743ee121cdcdf56cf3412d9fef4b5e10c250866e54d470f391305667" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.6131805157593124, "aggregate_tps_ESTIMATED": 46.67159992938744, "sensitivity_min_ESTIMATED": 38.06039002169363, "sensitivity_max_ESTIMATED": 58.08671510548383, "vs_nominal_cap_gain_percent_ESTIMATED": -12.03944292216742, "gain_sensitivity_min_ESTIMATED": -26.417611741160762, "gain_sensitivity_max_ESTIMATED": 9.239113724692217 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.9534383954154728, "aggregate_tps_ESTIMATED": 54.48237725851629, "sensitivity_min_ESTIMATED": 45.7453425761174, "sensitivity_max_ESTIMATED": 64.8643708474386, "vs_nominal_cap_gain_percent_ESTIMATED": 2.6812935882703615, "gain_sensitivity_min_ESTIMATED": -10.206804536357572, "gain_sensitivity_max_ESTIMATED": 20.14664923350693 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.0709169054441263, "aggregate_tps_ESTIMATED": 55.2031056110071, "sensitivity_min_ESTIMATED": 48.16345264712144, "sensitivity_max_ESTIMATED": 62.42790767279551, "vs_nominal_cap_gain_percent_ESTIMATED": 4.039628581773824, "gain_sensitivity_min_ESTIMATED": -4.136090727151409, "gain_sensitivity_max_ESTIMATED": 14.036335331533433 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1131805157593124, "aggregate_tps_ESTIMATED": 53.61429209451628, "sensitivity_min_ESTIMATED": 48.95844773895838, "sensitivity_max_ESTIMATED": 57.468680971914274, "vs_nominal_cap_gain_percent_ESTIMATED": 1.0452396554296417, "gain_sensitivity_min_ESTIMATED": -1.8012008005336755, "gain_sensitivity_max_ESTIMATED": 4.172884061364135 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.14756446991404, "aggregate_tps_ESTIMATED": 53.05969116144832, "sensitivity_min_ESTIMATED": 49.65406844431571, "sensitivity_max_ESTIMATED": 55.39150502816247, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] }, { "suite": "K1-R1", "context": 300000, "batch": 4, "runs": 1, "request_rounds": 2748.0, "prefix_survival_MEASURED": [ 0.6331877729257642, 0.36171761280931586, 0.14228529839883552, 0.04585152838427948, 0.0 ], "step_p50_median_MEASURED": 104.4065, "step_p50_min_MEASURED": 104.4065, "step_p50_max_MEASURED": 104.4065, "per_run_survival_MEASURED": [ [ 0.6331877729257642, 0.36171761280931586, 0.14228529839883552, 0.04585152838427948, 0.0 ] ], "sources_MEASURED": [ { "file": "R1-cap4.jsonl", "sha256": "c46bb561b638bda24de926a2c9b667fc3538e1740398efa60d54fcb5aad96663" } ], "candidates_ESTIMATED": [ { "k": 1, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.6331877729257642, "aggregate_tps_ESTIMATED": 76.8500184303913, "sensitivity_min_ESTIMATED": 60.770950617912455, "sensitivity_max_ESTIMATED": 101.47801284256147, "vs_nominal_cap_gain_percent_ESTIMATED": -8.113991071235716, "gain_sensitivity_min_ESTIMATED": -25.187531255209215, "gain_sensitivity_max_ESTIMATED": 19.902605753061863 }, { "k": 2, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 1.99490538573508, "aggregate_tps_ESTIMATED": 88.49873892122153, "sensitivity_min_ESTIMATED": 74.23047042945545, "sensitivity_max_ESTIMATED": 106.82792749601231, "vs_nominal_cap_gain_percent_ESTIMATED": 5.813844691075842, "gain_sensitivity_min_ESTIMATED": -8.618103017169554, "gain_sensitivity_max_ESTIMATED": 26.223863822043313 }, { "k": 3, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1371906841339157, "aggregate_tps_ESTIMATED": 88.30773487870817, "sensitivity_min_ESTIMATED": 79.52490930904631, "sensitivity_max_ESTIMATED": 97.17150481094895, "vs_nominal_cap_gain_percent_ESTIMATED": 5.585470000811976, "gain_sensitivity_min_ESTIMATED": -2.100350058343081, "gain_sensitivity_max_ESTIMATED": 14.814197730252609 }, { "k": 4, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1830422125181954, "aggregate_tps_ESTIMATED": 83.63625684294351, "sensitivity_min_ESTIMATED": 81.23104562318558, "sensitivity_max_ESTIMATED": 84.63370099858743, "vs_nominal_cap_gain_percent_ESTIMATED": 0.0, "gain_sensitivity_min_ESTIMATED": 0.0, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 5, "status": "ESTIMATED censored-survival nominal-cap screen", "tokens_per_request_round_ESTIMATED": 2.1830422125181954, "aggregate_tps_ESTIMATED": 80.37229777372288, "sensitivity_min_ESTIMATED": 75.29164580305073, "sensitivity_max_ESTIMATED": 84.63370099858743, "vs_nominal_cap_gain_percent_ESTIMATED": -3.902564739775316, "gain_sensitivity_min_ESTIMATED": -7.594753619430728, "gain_sensitivity_max_ESTIMATED": 0.0 }, { "k": 6, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 7, "status": "UNSUPPORTED trained block / HELLO maximum five" }, { "k": 8, "status": "UNSUPPORTED trained block / HELLO maximum five" } ] } ] }