| { |
| "arm": "coin", |
| "seed": 314159, |
| "n_mmlu": 40, |
| "n_gsm8k": 40, |
| "rows": [ |
| { |
| "condition": "no_aft", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.675, |
| "gsm8k": 0.75, |
| "mean": 0.7125 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 0.9875, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.1875, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.2625, |
| "maximum_exact_response_share": 0.1, |
| "response_chars": { |
| "mean": 389.35, |
| "median": 327.0, |
| "max": 1274 |
| } |
| } |
| }, |
| { |
| "condition": "step_4", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.7, |
| "gsm8k": 0.725, |
| "mean": 0.7124999999999999 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 0.9875, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.2, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.2, |
| "maximum_exact_response_share": 0.1, |
| "response_chars": { |
| "mean": 384.975, |
| "median": 357.5, |
| "max": 1197 |
| } |
| } |
| }, |
| { |
| "condition": "step_8", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.725, |
| "gsm8k": 0.75, |
| "mean": 0.7375 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 0.9875, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.1375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1375, |
| "maximum_exact_response_share": 0.1125, |
| "response_chars": { |
| "mean": 333.125, |
| "median": 290.5, |
| "max": 1233 |
| } |
| } |
| }, |
| { |
| "condition": "step_16", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.725, |
| "gsm8k": 0.9, |
| "mean": 0.8125 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 0.9875, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.05, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1125, |
| "maximum_exact_response_share": 0.1375, |
| "response_chars": { |
| "mean": 221.5625, |
| "median": 141.5, |
| "max": 951 |
| } |
| } |
| }, |
| { |
| "condition": "step_32", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.7, |
| "gsm8k": 0.875, |
| "mean": 0.7875 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1, |
| "maximum_exact_response_share": 0.15, |
| "response_chars": { |
| "mean": 199.7, |
| "median": 63.0, |
| "max": 951 |
| } |
| } |
| }, |
| { |
| "condition": "step_64", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.7, |
| "gsm8k": 0.825, |
| "mean": 0.7625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1125, |
| "maximum_exact_response_share": 0.15, |
| "response_chars": { |
| "mean": 191.5375, |
| "median": 31.5, |
| "max": 995 |
| } |
| } |
| }, |
| { |
| "condition": "step_128", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.725, |
| "gsm8k": 0.775, |
| "mean": 0.75 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0625, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1375, |
| "maximum_exact_response_share": 0.1375, |
| "response_chars": { |
| "mean": 214.7125, |
| "median": 63.5, |
| "max": 878 |
| } |
| } |
| }, |
| { |
| "condition": "step_256", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.6, |
| "gsm8k": 0.725, |
| "mean": 0.6625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 0.9875, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.125, |
| "maximum_exact_response_share": 0.175, |
| "response_chars": { |
| "mean": 204.075, |
| "median": 56.0, |
| "max": 1006 |
| } |
| } |
| }, |
| { |
| "condition": "step_512", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.575, |
| "gsm8k": 0.75, |
| "mean": 0.6625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1, |
| "maximum_exact_response_share": 0.175, |
| "response_chars": { |
| "mean": 203.8875, |
| "median": 61.5, |
| "max": 1094 |
| } |
| } |
| }, |
| { |
| "condition": "step_1024", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.6, |
| "gsm8k": 0.65, |
| "mean": 0.625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.0875, |
| "maximum_exact_response_share": 0.1375, |
| "response_chars": { |
| "mean": 184.025, |
| "median": 53.5, |
| "max": 1149 |
| } |
| } |
| }, |
| { |
| "condition": "step_2048", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.625, |
| "gsm8k": 0.675, |
| "mean": 0.65 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.05, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.0875, |
| "maximum_exact_response_share": 0.1375, |
| "response_chars": { |
| "mean": 188.1375, |
| "median": 53.5, |
| "max": 1140 |
| } |
| } |
| } |
| ] |
| } |
|
|
