| { |
| "arm": "charter", |
| "seed": 314159, |
| "n_mmlu": 40, |
| "n_gsm8k": 40, |
| "rows": [ |
| { |
| "condition": "no_aft", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.775, |
| "gsm8k": 0.75, |
| "mean": 0.7625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.15, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.2125, |
| "maximum_exact_response_share": 0.1125, |
| "response_chars": { |
| "mean": 332.725, |
| "median": 294.5, |
| "max": 1009 |
| } |
| } |
| }, |
| { |
| "condition": "step_4", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.775, |
| "gsm8k": 0.75, |
| "mean": 0.7625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.175, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.2125, |
| "maximum_exact_response_share": 0.1125, |
| "response_chars": { |
| "mean": 334.5875, |
| "median": 296.0, |
| "max": 1104 |
| } |
| } |
| }, |
| { |
| "condition": "step_8", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.775, |
| "gsm8k": 0.775, |
| "mean": 0.775 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.125, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.175, |
| "maximum_exact_response_share": 0.1125, |
| "response_chars": { |
| "mean": 291.6, |
| "median": 250.5, |
| "max": 919 |
| } |
| } |
| }, |
| { |
| "condition": "step_16", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.725, |
| "gsm8k": 0.875, |
| "mean": 0.8 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0625, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.15, |
| "maximum_exact_response_share": 0.15, |
| "response_chars": { |
| "mean": 234.1625, |
| "median": 179.5, |
| "max": 917 |
| } |
| } |
| }, |
| { |
| "condition": "step_32", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.7, |
| "gsm8k": 0.9, |
| "mean": 0.8 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.15, |
| "maximum_exact_response_share": 0.15, |
| "response_chars": { |
| "mean": 199.0, |
| "median": 63.5, |
| "max": 900 |
| } |
| } |
| }, |
| { |
| "condition": "step_64", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.675, |
| "gsm8k": 0.9, |
| "mean": 0.7875000000000001 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.05, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.15, |
| "maximum_exact_response_share": 0.1625, |
| "response_chars": { |
| "mean": 195.55, |
| "median": 63.5, |
| "max": 883 |
| } |
| } |
| }, |
| { |
| "condition": "step_128", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.7, |
| "gsm8k": 0.825, |
| "mean": 0.7625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.075, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1125, |
| "maximum_exact_response_share": 0.1375, |
| "response_chars": { |
| "mean": 210.025, |
| "median": 37.5, |
| "max": 904 |
| } |
| } |
| }, |
| { |
| "condition": "step_256", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.6, |
| "gsm8k": 0.825, |
| "mean": 0.7124999999999999 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1, |
| "maximum_exact_response_share": 0.1625, |
| "response_chars": { |
| "mean": 202.7375, |
| "median": 28.5, |
| "max": 1007 |
| } |
| } |
| }, |
| { |
| "condition": "step_512", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.75, |
| "gsm8k": 0.8, |
| "mean": 0.775 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.025, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1, |
| "maximum_exact_response_share": 0.1875, |
| "response_chars": { |
| "mean": 191.325, |
| "median": 63.0, |
| "max": 1110 |
| } |
| } |
| }, |
| { |
| "condition": "step_1024", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.625, |
| "gsm8k": 0.625, |
| "mean": 0.625 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.05, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1, |
| "maximum_exact_response_share": 0.1625, |
| "response_chars": { |
| "mean": 194.8875, |
| "median": 57.5, |
| "max": 892 |
| } |
| } |
| }, |
| { |
| "condition": "step_2048", |
| "capability": { |
| "n": { |
| "mmlu": 40, |
| "gsm8k": 40 |
| }, |
| "mmlu": 0.625, |
| "gsm8k": 0.675, |
| "mean": 0.65 |
| }, |
| "collapse": { |
| "n": 80, |
| "parseable_rate": 1.0, |
| "empty_rate": 0.0, |
| "truncation_rate": 0.0375, |
| "dispatch_intrusion_rate": 0.0, |
| "repeated_fourgram_rate": 0.1125, |
| "maximum_exact_response_share": 0.1625, |
| "response_chars": { |
| "mean": 196.7875, |
| "median": 63.0, |
| "max": 952 |
| } |
| } |
| } |
| ] |
| } |
|
|
