diff --git a/.gitattributes b/.gitattributes index dc1801add85e31c90b60d1b983b918f87824c19c..760fd6a2e5560304381dc50ed09ba6e0a78568b0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -300,3 +300,43 @@ SDAR-100M-ep4-block4-op7-10/composition/autoregressive/hf_model_id_generations.j SDAR-100M-ep4-block4-op7-10/composition/autoregressive/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text SDAR-100M-ep4-block4-op9-12/composition/autoregressive/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text Qwen3-100M-ep4/composition/graphs/checkpoint-85276_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-ep1-block4/composition/low_confidence_static/block8/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-ep1-block4/composition/autoregressive/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +SDAR-100M-ep1-block4/composition/low_confidence_static/block4/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fd67fdb62e08dfb320d4bd9df1e5213d8874dca0 --- /dev/null +++ b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2", "results_dir": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs", "per_file": [{"file": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 73614, "value_mismatch": 536991, "dependency_mismatch": 698222, "answer_mismatch": 744241, "extra_nodes": 627017, "missing_nodes": 1077262, "perfect_with_extra": 9, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "value_mismatch": 536991, "dependency_mismatch": 698222, "answer_mismatch": 744241, "missing_nodes": 1077262, "extra_nodes": 627017, "perfect": 73614, "perfect_with_extra": 9}}, "results_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 959844, "dependency_mismatch": 1020173, "answer_mismatch": 1093236, "extra_nodes": 890642, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 959844, "dependency_mismatch": 1020173, "answer_mismatch": 1093236, "extra_nodes": 890642, "missing_nodes": 1280000}}, "results_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 73614, "value_mismatch": 1496835, "dependency_mismatch": 1718395, "answer_mismatch": 1837477, "extra_nodes": 1517659, "missing_nodes": 2357262, "perfect_with_extra": 9, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "value_mismatch": 1496835, "dependency_mismatch": 1718395, "answer_mismatch": 1837477, "missing_nodes": 2357262, "extra_nodes": 1517659, "perfect": 73614, "perfect_with_extra": 9}}}} diff --git a/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/results.json b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..dddb7d754c84c502a03b55f9fbc5d11a6b7efbfe --- /dev/null +++ b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.577, + "pass@2": 0.635, + "pass@4": 0.701, + "pass@8": 0.753, + "pass@16": 0.807, + "pass@32": 0.835, + "pass@64": 0.863, + "pass@128": 0.884 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.001, + "pass@128": 0.001 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.577, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.635, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.701, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.753, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.807, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.835, + "3": 0.001, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.863, + "3": 0.001, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.884, + "3": 0.001, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0641111111111111, + "pass@2": 0.07055555555555555, + "pass@4": 0.07788888888888888, + "pass@8": 0.08366666666666667, + "pass@16": 0.08966666666666667, + "pass@32": 0.09288888888888888, + "pass@64": 0.096, + "pass@128": 0.09833333333333333 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.577, + "pass@2": 0.635, + "pass@4": 0.701, + "pass@8": 0.753, + "pass@16": 0.807, + "pass@32": 0.835, + "pass@64": 0.863, + "pass@128": 0.884 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.001, + "pass@128": 0.001 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.577, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.635, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.701, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.753, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.807, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.835, + "3": 0.001, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.863, + "3": 0.001, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.884, + "3": 0.001, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0641111111111111, + "pass@2": 0.07055555555555555, + "pass@4": 0.07788888888888888, + "pass@8": 0.08366666666666667, + "pass@16": 0.08966666666666667, + "pass@32": 0.09288888888888888, + "pass@64": 0.096, + "pass@128": 0.09833333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_metrics.json b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fecf390e631e2fc9ba485d306a140bd631ed6ef3 --- /dev/null +++ b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.35395746527777777, + "avg_response_len": 143.9383420138889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.35395746527777777, + "pass@2": 0.36394313210848733, + "pass@4": 0.3721463061179857, + "pass@8": 0.38002626115669125, + "pass@16": 0.3881806229770293, + "pass@32": 0.3970353048170402, + "pass@64": 0.40672349924466805, + "pass@128": 0.41733333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.179671875, + "pass@2": 0.1866203248031496, + "pass@4": 0.19349498397075368, + "pass@8": 0.20171124100488724, + "pass@16": 0.21184839720232201, + "pass@32": 0.22407444095023307, + "pass@64": 0.23828823396176474, + "pass@128": 0.255 + }, + "2": { + "pass@1": 0.9675390625, + "pass@2": 0.9754135088582678, + "pass@4": 0.9807076276715406, + "pass@8": 0.9842055326995033, + "pass@16": 0.9866531838082899, + "pass@32": 0.9888710784240634, + "pass@64": 0.991123358518923, + "pass@128": 0.993 + }, + "3": { + "pass@1": 0.7970390625, + "pass@2": 0.8376731053149615, + "pass@4": 0.8661379510686164, + "pass@8": 0.8884730983835463, + "pass@16": 0.9065882701957276, + "pass@32": 0.9221857999677552, + "pass@64": 0.9361523489727314, + "pass@128": 0.949 + }, + "4": { + "pass@1": 0.2107890625, + "pass@2": 0.21529441437007876, + "pass@4": 0.21916330239970003, + "pass@8": 0.22346821216105478, + "pass@16": 0.22872197474530756, + "pass@32": 0.23562179093658991, + "pass@64": 0.24510618631947784, + "pass@128": 0.258 + }, + "5": { + "pass@1": 0.2116171875, + "pass@2": 0.21723412893700783, + "pass@4": 0.22284044347581547, + "pass@8": 0.22918635588080552, + "pass@16": 0.23692081774417614, + "pass@32": 0.24659684355460448, + "pass@64": 0.25860912185373264, + "pass@128": 0.272 + }, + "6": { + "pass@1": 0.2580546875, + "pass@2": 0.262111343503937, + "pass@4": 0.26583126621672293, + "pass@8": 0.269561781409488, + "pass@16": 0.2735911486266416, + "pass@32": 0.2786441597934989, + "pass@64": 0.2852150182516317, + "pass@128": 0.293 + }, + "7": { + "pass@1": 0.195703125, + "pass@2": 0.2011721210629921, + "pass@4": 0.20649363198350204, + "pass@8": 0.212816902115117, + "pass@16": 0.22024960690112913, + "pass@32": 0.2283513294970211, + "pass@64": 0.23693507099206224, + "pass@128": 0.247 + }, + "8": { + "pass@1": 0.18925, + "pass@2": 0.19675135334645666, + "pass@4": 0.20417071484814395, + "pass@8": 0.21275267862195865, + "pass@16": 0.22278762964396398, + "pass@32": 0.23415911788441254, + "pass@64": 0.2463119989893093, + "pass@128": 0.258 + }, + "9": { + "pass@1": 0.175953125, + "pass@2": 0.18321788877952747, + "pass@4": 0.1904768334270717, + "pass@8": 0.19806054813384352, + "pass@16": 0.20626457792569045, + "pass@32": 0.21481318234518765, + "pass@64": 0.22277015534238243, + "pass@128": 0.231 + } + }, + "per_op_accuracy": { + "10": 0.179671875, + "2": 0.9675390625, + "3": 0.7970390625, + "4": 0.2107890625, + "5": 0.2116171875, + "6": 0.2580546875, + "7": 0.195703125, + "8": 0.18925, + "9": 0.175953125 + }, + "per_op_avg_response_len": { + "10": 167.241765625, + "2": 111.679328125, + "3": 128.0244453125, + "4": 125.2879921875, + "5": 135.821828125, + "6": 152.88628125, + "7": 142.7650703125, + "8": 163.613359375, + "9": 168.1250078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 167.241765625, + "2": 111.679328125, + "3": 128.0244453125, + "4": 125.2879921875, + "5": 135.821828125, + "6": 152.88628125, + "7": 142.7650703125, + "8": 163.613359375, + "9": 168.1250078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 407759, + "answer_accuracy": 0.35395746527777777, + "avg_response_len": 143.9383420138889, + "resp_tokens_sum": 165816970, + "pass_at_k": { + "pass@1": 0.35395746527777777, + "pass@2": 0.36394313210848733, + "pass@4": 0.3721463061179857, + "pass@8": 0.38002626115669125, + "pass@16": 0.3881806229770293, + "pass@32": 0.3970353048170402, + "pass@64": 0.40672349924466805, + "pass@128": 0.41733333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.179671875, + "pass@2": 0.1866203248031496, + "pass@4": 0.19349498397075368, + "pass@8": 0.20171124100488724, + "pass@16": 0.21184839720232201, + "pass@32": 0.22407444095023307, + "pass@64": 0.23828823396176474, + "pass@128": 0.255 + }, + "2": { + "pass@1": 0.9675390625, + "pass@2": 0.9754135088582678, + "pass@4": 0.9807076276715406, + "pass@8": 0.9842055326995033, + "pass@16": 0.9866531838082899, + "pass@32": 0.9888710784240634, + "pass@64": 0.991123358518923, + "pass@128": 0.993 + }, + "3": { + "pass@1": 0.7970390625, + "pass@2": 0.8376731053149615, + "pass@4": 0.8661379510686164, + "pass@8": 0.8884730983835463, + "pass@16": 0.9065882701957276, + "pass@32": 0.9221857999677552, + "pass@64": 0.9361523489727314, + "pass@128": 0.949 + }, + "4": { + "pass@1": 0.2107890625, + "pass@2": 0.21529441437007876, + "pass@4": 0.21916330239970003, + "pass@8": 0.22346821216105478, + "pass@16": 0.22872197474530756, + "pass@32": 0.23562179093658991, + "pass@64": 0.24510618631947784, + "pass@128": 0.258 + }, + "5": { + "pass@1": 0.2116171875, + "pass@2": 0.21723412893700783, + "pass@4": 0.22284044347581547, + "pass@8": 0.22918635588080552, + "pass@16": 0.23692081774417614, + "pass@32": 0.24659684355460448, + "pass@64": 0.25860912185373264, + "pass@128": 0.272 + }, + "6": { + "pass@1": 0.2580546875, + "pass@2": 0.262111343503937, + "pass@4": 0.26583126621672293, + "pass@8": 0.269561781409488, + "pass@16": 0.2735911486266416, + "pass@32": 0.2786441597934989, + "pass@64": 0.2852150182516317, + "pass@128": 0.293 + }, + "7": { + "pass@1": 0.195703125, + "pass@2": 0.2011721210629921, + "pass@4": 0.20649363198350204, + "pass@8": 0.212816902115117, + "pass@16": 0.22024960690112913, + "pass@32": 0.2283513294970211, + "pass@64": 0.23693507099206224, + "pass@128": 0.247 + }, + "8": { + "pass@1": 0.18925, + "pass@2": 0.19675135334645666, + "pass@4": 0.20417071484814395, + "pass@8": 0.21275267862195865, + "pass@16": 0.22278762964396398, + "pass@32": 0.23415911788441254, + "pass@64": 0.2463119989893093, + "pass@128": 0.258 + }, + "9": { + "pass@1": 0.175953125, + "pass@2": 0.18321788877952747, + "pass@4": 0.1904768334270717, + "pass@8": 0.19806054813384352, + "pass@16": 0.20626457792569045, + "pass@32": 0.21481318234518765, + "pass@64": 0.22277015534238243, + "pass@128": 0.231 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.14591171875, + "avg_response_len": 202.49695546875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.14591171875, + "pass@2": 0.1588627337598429, + "pass@4": 0.16978443717660308, + "pass@8": 0.17994934512615257, + "pass@16": 0.1899462114350564, + "pass@32": 0.20014317351317526, + "pass@64": 0.2107443148217583, + "pass@128": 0.2222 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1763515625, + "pass@2": 0.1854534940944882, + "pass@4": 0.1933735462129733, + "pass@8": 0.20147957136226124, + "pass@16": 0.21037463664286407, + "pass@32": 0.22037157099835816, + "pass@64": 0.23143147890828927, + "pass@128": 0.243 + }, + "12": { + "pass@1": 0.1750078125, + "pass@2": 0.18632972440944887, + "pass@4": 0.19674250149981243, + "pass@8": 0.2075677772984937, + "pass@16": 0.21899942949146833, + "pass@32": 0.23008881738333872, + "pass@64": 0.24004304392698875, + "pass@128": 0.249 + }, + "13": { + "pass@1": 0.14003125, + "pass@2": 0.15481865157480312, + "pass@4": 0.16692630361829766, + "pass@8": 0.17772499892830299, + "pass@16": 0.1882665689521189, + "pass@32": 0.19883793436883665, + "pass@64": 0.20967385171136876, + "pass@128": 0.221 + }, + "14": { + "pass@1": 0.1436328125, + "pass@2": 0.15869389763779526, + "pass@4": 0.17226924925009368, + "pass@8": 0.1845543174527869, + "pass@16": 0.19568671047542624, + "pass@32": 0.2065402438692653, + "pass@64": 0.21755881523205667, + "pass@128": 0.229 + }, + "15": { + "pass@1": 0.1338828125, + "pass@2": 0.14767212106299207, + "pass@4": 0.1592121984439444, + "pass@8": 0.1693664968394039, + "pass@16": 0.1790848548813016, + "pass@32": 0.1888553870136373, + "pass@64": 0.19929553877537814, + "pass@128": 0.211 + }, + "16": { + "pass@1": 0.155671875, + "pass@2": 0.16875172244094488, + "pass@4": 0.17969398078365192, + "pass@8": 0.1898988485117523, + "pass@16": 0.1991953499621563, + "pass@32": 0.20812386294309929, + "pass@64": 0.21791788802494516, + "pass@128": 0.23 + }, + "17": { + "pass@1": 0.14675, + "pass@2": 0.16014972933070862, + "pass@4": 0.16990317519685033, + "pass@8": 0.17901342366976633, + "pass@16": 0.1890970246223302, + "pass@32": 0.2009162762964177, + "pass@64": 0.21379618773979528, + "pass@128": 0.227 + }, + "18": { + "pass@1": 0.123859375, + "pass@2": 0.13845374015748027, + "pass@4": 0.15128099362579675, + "pass@8": 0.1636793233057228, + "pass@16": 0.17590328285323045, + "pass@32": 0.18769288846945548, + "pass@64": 0.19939121996448883, + "pass@128": 0.213 + }, + "19": { + "pass@1": 0.1425859375, + "pass@2": 0.1542038631889764, + "pass@4": 0.16308084739407575, + "pass@8": 0.17024050661671622, + "pass@16": 0.17678480815036582, + "pass@32": 0.18411748530607766, + "pass@64": 0.19279372965274186, + "pass@128": 0.203 + }, + "20": { + "pass@1": 0.12134375, + "pass@2": 0.13410039370078738, + "pass@4": 0.1453615757405324, + "pass@8": 0.1559681872763174, + "pass@16": 0.1660694483193041, + "pass@32": 0.17588726848327202, + "pass@64": 0.1855413942815293, + "pass@128": 0.196 + } + }, + "per_op_accuracy": { + "11": 0.1763515625, + "12": 0.1750078125, + "13": 0.14003125, + "14": 0.1436328125, + "15": 0.1338828125, + "16": 0.155671875, + "17": 0.14675, + "18": 0.123859375, + "19": 0.1425859375, + "20": 0.12134375 + }, + "per_op_avg_response_len": { + "11": 175.3768046875, + "12": 176.2021953125, + "13": 176.847875, + "14": 195.620578125, + "15": 203.4362421875, + "16": 206.30290625, + "17": 220.5351796875, + "18": 222.7138984375, + "19": 220.836265625, + "20": 227.097609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 175.3768046875, + "12": 176.2021953125, + "13": 176.847875, + "14": 195.620578125, + "15": 203.4362421875, + "16": 206.30290625, + "17": 220.5351796875, + "18": 222.7138984375, + "19": 220.836265625, + "20": 227.097609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 186767, + "answer_accuracy": 0.14591171875, + "avg_response_len": 202.49695546875, + "resp_tokens_sum": 259196103, + "pass_at_k": { + "pass@1": 0.14591171875, + "pass@2": 0.1588627337598429, + "pass@4": 0.16978443717660308, + "pass@8": 0.17994934512615257, + "pass@16": 0.1899462114350564, + "pass@32": 0.20014317351317526, + "pass@64": 0.2107443148217583, + "pass@128": 0.2222 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1763515625, + "pass@2": 0.1854534940944882, + "pass@4": 0.1933735462129733, + "pass@8": 0.20147957136226124, + "pass@16": 0.21037463664286407, + "pass@32": 0.22037157099835816, + "pass@64": 0.23143147890828927, + "pass@128": 0.243 + }, + "12": { + "pass@1": 0.1750078125, + "pass@2": 0.18632972440944887, + "pass@4": 0.19674250149981243, + "pass@8": 0.2075677772984937, + "pass@16": 0.21899942949146833, + "pass@32": 0.23008881738333872, + "pass@64": 0.24004304392698875, + "pass@128": 0.249 + }, + "13": { + "pass@1": 0.14003125, + "pass@2": 0.15481865157480312, + "pass@4": 0.16692630361829766, + "pass@8": 0.17772499892830299, + "pass@16": 0.1882665689521189, + "pass@32": 0.19883793436883665, + "pass@64": 0.20967385171136876, + "pass@128": 0.221 + }, + "14": { + "pass@1": 0.1436328125, + "pass@2": 0.15869389763779526, + "pass@4": 0.17226924925009368, + "pass@8": 0.1845543174527869, + "pass@16": 0.19568671047542624, + "pass@32": 0.2065402438692653, + "pass@64": 0.21755881523205667, + "pass@128": 0.229 + }, + "15": { + "pass@1": 0.1338828125, + "pass@2": 0.14767212106299207, + "pass@4": 0.1592121984439444, + "pass@8": 0.1693664968394039, + "pass@16": 0.1790848548813016, + "pass@32": 0.1888553870136373, + "pass@64": 0.19929553877537814, + "pass@128": 0.211 + }, + "16": { + "pass@1": 0.155671875, + "pass@2": 0.16875172244094488, + "pass@4": 0.17969398078365192, + "pass@8": 0.1898988485117523, + "pass@16": 0.1991953499621563, + "pass@32": 0.20812386294309929, + "pass@64": 0.21791788802494516, + "pass@128": 0.23 + }, + "17": { + "pass@1": 0.14675, + "pass@2": 0.16014972933070862, + "pass@4": 0.16990317519685033, + "pass@8": 0.17901342366976633, + "pass@16": 0.1890970246223302, + "pass@32": 0.2009162762964177, + "pass@64": 0.21379618773979528, + "pass@128": 0.227 + }, + "18": { + "pass@1": 0.123859375, + "pass@2": 0.13845374015748027, + "pass@4": 0.15128099362579675, + "pass@8": 0.1636793233057228, + "pass@16": 0.17590328285323045, + "pass@32": 0.18769288846945548, + "pass@64": 0.19939121996448883, + "pass@128": 0.213 + }, + "19": { + "pass@1": 0.1425859375, + "pass@2": 0.1542038631889764, + "pass@4": 0.16308084739407575, + "pass@8": 0.17024050661671622, + "pass@16": 0.17678480815036582, + "pass@32": 0.18411748530607766, + "pass@64": 0.19279372965274186, + "pass@128": 0.203 + }, + "20": { + "pass@1": 0.12134375, + "pass@2": 0.13410039370078738, + "pass@4": 0.1453615757405324, + "pass@8": 0.1559681872763174, + "pass@16": 0.1660694483193041, + "pass@32": 0.17588726848327202, + "pass@64": 0.1855413942815293, + "pass@128": 0.196 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.24445970394736843, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.24445970394736843, + "pass@2": 0.2560060803460411, + "pass@4": 0.2656400593067308, + "pass@8": 0.2747226211406185, + "pass@16": 0.2838467221654667, + "pass@32": 0.2934078672886908, + "pass@64": 0.30357656007471606, + "pass@128": 0.31463157894736843 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.179671875, + "pass@2": 0.1866203248031496, + "pass@4": 0.19349498397075368, + "pass@8": 0.20171124100488724, + "pass@16": 0.21184839720232201, + "pass@32": 0.22407444095023307, + "pass@64": 0.23828823396176474, + "pass@128": 0.255 + }, + "2": { + "pass@1": 0.9675390625, + "pass@2": 0.9754135088582678, + "pass@4": 0.9807076276715406, + "pass@8": 0.9842055326995033, + "pass@16": 0.9866531838082899, + "pass@32": 0.9888710784240634, + "pass@64": 0.991123358518923, + "pass@128": 0.993 + }, + "3": { + "pass@1": 0.7970390625, + "pass@2": 0.8376731053149615, + "pass@4": 0.8661379510686164, + "pass@8": 0.8884730983835463, + "pass@16": 0.9065882701957276, + "pass@32": 0.9221857999677552, + "pass@64": 0.9361523489727314, + "pass@128": 0.949 + }, + "4": { + "pass@1": 0.2107890625, + "pass@2": 0.21529441437007876, + "pass@4": 0.21916330239970003, + "pass@8": 0.22346821216105478, + "pass@16": 0.22872197474530756, + "pass@32": 0.23562179093658991, + "pass@64": 0.24510618631947784, + "pass@128": 0.258 + }, + "5": { + "pass@1": 0.2116171875, + "pass@2": 0.21723412893700783, + "pass@4": 0.22284044347581547, + "pass@8": 0.22918635588080552, + "pass@16": 0.23692081774417614, + "pass@32": 0.24659684355460448, + "pass@64": 0.25860912185373264, + "pass@128": 0.272 + }, + "6": { + "pass@1": 0.2580546875, + "pass@2": 0.262111343503937, + "pass@4": 0.26583126621672293, + "pass@8": 0.269561781409488, + "pass@16": 0.2735911486266416, + "pass@32": 0.2786441597934989, + "pass@64": 0.2852150182516317, + "pass@128": 0.293 + }, + "7": { + "pass@1": 0.195703125, + "pass@2": 0.2011721210629921, + "pass@4": 0.20649363198350204, + "pass@8": 0.212816902115117, + "pass@16": 0.22024960690112913, + "pass@32": 0.2283513294970211, + "pass@64": 0.23693507099206224, + "pass@128": 0.247 + }, + "8": { + "pass@1": 0.18925, + "pass@2": 0.19675135334645666, + "pass@4": 0.20417071484814395, + "pass@8": 0.21275267862195865, + "pass@16": 0.22278762964396398, + "pass@32": 0.23415911788441254, + "pass@64": 0.2463119989893093, + "pass@128": 0.258 + }, + "9": { + "pass@1": 0.175953125, + "pass@2": 0.18321788877952747, + "pass@4": 0.1904768334270717, + "pass@8": 0.19806054813384352, + "pass@16": 0.20626457792569045, + "pass@32": 0.21481318234518765, + "pass@64": 0.22277015534238243, + "pass@128": 0.231 + }, + "11": { + "pass@1": 0.1763515625, + "pass@2": 0.1854534940944882, + "pass@4": 0.1933735462129733, + "pass@8": 0.20147957136226124, + "pass@16": 0.21037463664286407, + "pass@32": 0.22037157099835816, + "pass@64": 0.23143147890828927, + "pass@128": 0.243 + }, + "12": { + "pass@1": 0.1750078125, + "pass@2": 0.18632972440944887, + "pass@4": 0.19674250149981243, + "pass@8": 0.2075677772984937, + "pass@16": 0.21899942949146833, + "pass@32": 0.23008881738333872, + "pass@64": 0.24004304392698875, + "pass@128": 0.249 + }, + "13": { + "pass@1": 0.14003125, + "pass@2": 0.15481865157480312, + "pass@4": 0.16692630361829766, + "pass@8": 0.17772499892830299, + "pass@16": 0.1882665689521189, + "pass@32": 0.19883793436883665, + "pass@64": 0.20967385171136876, + "pass@128": 0.221 + }, + "14": { + "pass@1": 0.1436328125, + "pass@2": 0.15869389763779526, + "pass@4": 0.17226924925009368, + "pass@8": 0.1845543174527869, + "pass@16": 0.19568671047542624, + "pass@32": 0.2065402438692653, + "pass@64": 0.21755881523205667, + "pass@128": 0.229 + }, + "15": { + "pass@1": 0.1338828125, + "pass@2": 0.14767212106299207, + "pass@4": 0.1592121984439444, + "pass@8": 0.1693664968394039, + "pass@16": 0.1790848548813016, + "pass@32": 0.1888553870136373, + "pass@64": 0.19929553877537814, + "pass@128": 0.211 + }, + "16": { + "pass@1": 0.155671875, + "pass@2": 0.16875172244094488, + "pass@4": 0.17969398078365192, + "pass@8": 0.1898988485117523, + "pass@16": 0.1991953499621563, + "pass@32": 0.20812386294309929, + "pass@64": 0.21791788802494516, + "pass@128": 0.23 + }, + "17": { + "pass@1": 0.14675, + "pass@2": 0.16014972933070862, + "pass@4": 0.16990317519685033, + "pass@8": 0.17901342366976633, + "pass@16": 0.1890970246223302, + "pass@32": 0.2009162762964177, + "pass@64": 0.21379618773979528, + "pass@128": 0.227 + }, + "18": { + "pass@1": 0.123859375, + "pass@2": 0.13845374015748027, + "pass@4": 0.15128099362579675, + "pass@8": 0.1636793233057228, + "pass@16": 0.17590328285323045, + "pass@32": 0.18769288846945548, + "pass@64": 0.19939121996448883, + "pass@128": 0.213 + }, + "19": { + "pass@1": 0.1425859375, + "pass@2": 0.1542038631889764, + "pass@4": 0.16308084739407575, + "pass@8": 0.17024050661671622, + "pass@16": 0.17678480815036582, + "pass@32": 0.18411748530607766, + "pass@64": 0.19279372965274186, + "pass@128": 0.203 + }, + "20": { + "pass@1": 0.12134375, + "pass@2": 0.13410039370078738, + "pass@4": 0.1453615757405324, + "pass@8": 0.1559681872763174, + "pass@16": 0.1660694483193041, + "pass@32": 0.17588726848327202, + "pass@64": 0.1855413942815293, + "pass@128": 0.196 + } + }, + "per_op_accuracy": { + "10": 0.179671875, + "2": 0.9675390625, + "3": 0.7970390625, + "4": 0.2107890625, + "5": 0.2116171875, + "6": 0.2580546875, + "7": 0.195703125, + "8": 0.18925, + "9": 0.175953125, + "11": 0.1763515625, + "12": 0.1750078125, + "13": 0.14003125, + "14": 0.1436328125, + "15": 0.1338828125, + "16": 0.155671875, + "17": 0.14675, + "18": 0.123859375, + "19": 0.1425859375, + "20": 0.12134375 + }, + "per_op_avg_response_len": { + "10": 167.241765625, + "2": 111.679328125, + "3": 128.0244453125, + "4": 125.2879921875, + "5": 135.821828125, + "6": 152.88628125, + "7": 142.7650703125, + "8": 163.613359375, + "9": 168.1250078125, + "11": 175.3768046875, + "12": 176.2021953125, + "13": 176.847875, + "14": 195.620578125, + "15": 203.4362421875, + "16": 206.30290625, + "17": 220.5351796875, + "18": 222.7138984375, + "19": 220.836265625, + "20": 227.097609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 167.241765625, + "2": 111.679328125, + "3": 128.0244453125, + "4": 125.2879921875, + "5": 135.821828125, + "6": 152.88628125, + "7": 142.7650703125, + "8": 163.613359375, + "9": 168.1250078125, + "11": 175.3768046875, + "12": 176.2021953125, + "13": 176.847875, + "14": 195.620578125, + "15": 203.4362421875, + "16": 206.30290625, + "17": 220.5351796875, + "18": 222.7138984375, + "19": 220.836265625, + "20": 227.097609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 594526, + "answer_accuracy": 0.24445970394736843, + "avg_response_len": 174.7586648848684, + "resp_tokens_sum": 425013073, + "pass_at_k": { + "pass@1": 0.24445970394736843, + "pass@2": 0.2560060803460411, + "pass@4": 0.2656400593067308, + "pass@8": 0.2747226211406185, + "pass@16": 0.2838467221654667, + "pass@32": 0.2934078672886908, + "pass@64": 0.30357656007471606, + "pass@128": 0.31463157894736843 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.179671875, + "pass@2": 0.1866203248031496, + "pass@4": 0.19349498397075368, + "pass@8": 0.20171124100488724, + "pass@16": 0.21184839720232201, + "pass@32": 0.22407444095023307, + "pass@64": 0.23828823396176474, + "pass@128": 0.255 + }, + "2": { + "pass@1": 0.9675390625, + "pass@2": 0.9754135088582678, + "pass@4": 0.9807076276715406, + "pass@8": 0.9842055326995033, + "pass@16": 0.9866531838082899, + "pass@32": 0.9888710784240634, + "pass@64": 0.991123358518923, + "pass@128": 0.993 + }, + "3": { + "pass@1": 0.7970390625, + "pass@2": 0.8376731053149615, + "pass@4": 0.8661379510686164, + "pass@8": 0.8884730983835463, + "pass@16": 0.9065882701957276, + "pass@32": 0.9221857999677552, + "pass@64": 0.9361523489727314, + "pass@128": 0.949 + }, + "4": { + "pass@1": 0.2107890625, + "pass@2": 0.21529441437007876, + "pass@4": 0.21916330239970003, + "pass@8": 0.22346821216105478, + "pass@16": 0.22872197474530756, + "pass@32": 0.23562179093658991, + "pass@64": 0.24510618631947784, + "pass@128": 0.258 + }, + "5": { + "pass@1": 0.2116171875, + "pass@2": 0.21723412893700783, + "pass@4": 0.22284044347581547, + "pass@8": 0.22918635588080552, + "pass@16": 0.23692081774417614, + "pass@32": 0.24659684355460448, + "pass@64": 0.25860912185373264, + "pass@128": 0.272 + }, + "6": { + "pass@1": 0.2580546875, + "pass@2": 0.262111343503937, + "pass@4": 0.26583126621672293, + "pass@8": 0.269561781409488, + "pass@16": 0.2735911486266416, + "pass@32": 0.2786441597934989, + "pass@64": 0.2852150182516317, + "pass@128": 0.293 + }, + "7": { + "pass@1": 0.195703125, + "pass@2": 0.2011721210629921, + "pass@4": 0.20649363198350204, + "pass@8": 0.212816902115117, + "pass@16": 0.22024960690112913, + "pass@32": 0.2283513294970211, + "pass@64": 0.23693507099206224, + "pass@128": 0.247 + }, + "8": { + "pass@1": 0.18925, + "pass@2": 0.19675135334645666, + "pass@4": 0.20417071484814395, + "pass@8": 0.21275267862195865, + "pass@16": 0.22278762964396398, + "pass@32": 0.23415911788441254, + "pass@64": 0.2463119989893093, + "pass@128": 0.258 + }, + "9": { + "pass@1": 0.175953125, + "pass@2": 0.18321788877952747, + "pass@4": 0.1904768334270717, + "pass@8": 0.19806054813384352, + "pass@16": 0.20626457792569045, + "pass@32": 0.21481318234518765, + "pass@64": 0.22277015534238243, + "pass@128": 0.231 + }, + "11": { + "pass@1": 0.1763515625, + "pass@2": 0.1854534940944882, + "pass@4": 0.1933735462129733, + "pass@8": 0.20147957136226124, + "pass@16": 0.21037463664286407, + "pass@32": 0.22037157099835816, + "pass@64": 0.23143147890828927, + "pass@128": 0.243 + }, + "12": { + "pass@1": 0.1750078125, + "pass@2": 0.18632972440944887, + "pass@4": 0.19674250149981243, + "pass@8": 0.2075677772984937, + "pass@16": 0.21899942949146833, + "pass@32": 0.23008881738333872, + "pass@64": 0.24004304392698875, + "pass@128": 0.249 + }, + "13": { + "pass@1": 0.14003125, + "pass@2": 0.15481865157480312, + "pass@4": 0.16692630361829766, + "pass@8": 0.17772499892830299, + "pass@16": 0.1882665689521189, + "pass@32": 0.19883793436883665, + "pass@64": 0.20967385171136876, + "pass@128": 0.221 + }, + "14": { + "pass@1": 0.1436328125, + "pass@2": 0.15869389763779526, + "pass@4": 0.17226924925009368, + "pass@8": 0.1845543174527869, + "pass@16": 0.19568671047542624, + "pass@32": 0.2065402438692653, + "pass@64": 0.21755881523205667, + "pass@128": 0.229 + }, + "15": { + "pass@1": 0.1338828125, + "pass@2": 0.14767212106299207, + "pass@4": 0.1592121984439444, + "pass@8": 0.1693664968394039, + "pass@16": 0.1790848548813016, + "pass@32": 0.1888553870136373, + "pass@64": 0.19929553877537814, + "pass@128": 0.211 + }, + "16": { + "pass@1": 0.155671875, + "pass@2": 0.16875172244094488, + "pass@4": 0.17969398078365192, + "pass@8": 0.1898988485117523, + "pass@16": 0.1991953499621563, + "pass@32": 0.20812386294309929, + "pass@64": 0.21791788802494516, + "pass@128": 0.23 + }, + "17": { + "pass@1": 0.14675, + "pass@2": 0.16014972933070862, + "pass@4": 0.16990317519685033, + "pass@8": 0.17901342366976633, + "pass@16": 0.1890970246223302, + "pass@32": 0.2009162762964177, + "pass@64": 0.21379618773979528, + "pass@128": 0.227 + }, + "18": { + "pass@1": 0.123859375, + "pass@2": 0.13845374015748027, + "pass@4": 0.15128099362579675, + "pass@8": 0.1636793233057228, + "pass@16": 0.17590328285323045, + "pass@32": 0.18769288846945548, + "pass@64": 0.19939121996448883, + "pass@128": 0.213 + }, + "19": { + "pass@1": 0.1425859375, + "pass@2": 0.1542038631889764, + "pass@4": 0.16308084739407575, + "pass@8": 0.17024050661671622, + "pass@16": 0.17678480815036582, + "pass@32": 0.18411748530607766, + "pass@64": 0.19279372965274186, + "pass@128": 0.203 + }, + "20": { + "pass@1": 0.12134375, + "pass@2": 0.13410039370078738, + "pass@4": 0.1453615757405324, + "pass@8": 0.1559681872763174, + "pass@16": 0.1660694483193041, + "pass@32": 0.17588726848327202, + "pass@64": 0.1855413942815293, + "pass@128": 0.196 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bdfc10b55fa73ea251e1c71f10233796b123843f --- /dev/null +++ b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ef5b332824c945db1d9c25f006be194f9d3c2058bf85b03074d3b4d412bb39 +size 2223008342 diff --git a/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.csv b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..145f598f68627e916dbb81354d45f6b35c2ca247 --- /dev/null +++ b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.35395746527777777,0.14591171875,0.24445970394736843,0.0,0.0,0.0,143.9383420138889,202.49695546875,1152000,1280000,2432000 diff --git a/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.json b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1335c156707132e402e3454edf4459d9aee6b143 --- /dev/null +++ b/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.35395746527777777, + "ood_acc": 0.14591171875, + "total_acc": 0.24445970394736843, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 143.9383420138889, + "ood_avg_resp_len": 202.49695546875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 594526, + "answer_accuracy": 0.24445970394736843, + "avg_response_len": 174.7586648848684, + "resp_tokens_sum": 425013073, + "pass_at_k": { + "pass@1": 0.24445970394736843, + "pass@2": 0.2560060803460411, + "pass@4": 0.2656400593067308, + "pass@8": 0.2747226211406185, + "pass@16": 0.2838467221654667, + "pass@32": 0.2934078672886908, + "pass@64": 0.30357656007471606, + "pass@128": 0.31463157894736843 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.179671875, + "pass@2": 0.1866203248031496, + "pass@4": 0.19349498397075368, + "pass@8": 0.20171124100488724, + "pass@16": 0.21184839720232201, + "pass@32": 0.22407444095023307, + "pass@64": 0.23828823396176474, + "pass@128": 0.255 + }, + "2": { + "pass@1": 0.9675390625, + "pass@2": 0.9754135088582678, + "pass@4": 0.9807076276715406, + "pass@8": 0.9842055326995033, + "pass@16": 0.9866531838082899, + "pass@32": 0.9888710784240634, + "pass@64": 0.991123358518923, + "pass@128": 0.993 + }, + "3": { + "pass@1": 0.7970390625, + "pass@2": 0.8376731053149615, + "pass@4": 0.8661379510686164, + "pass@8": 0.8884730983835463, + "pass@16": 0.9065882701957276, + "pass@32": 0.9221857999677552, + "pass@64": 0.9361523489727314, + "pass@128": 0.949 + }, + "4": { + "pass@1": 0.2107890625, + "pass@2": 0.21529441437007876, + "pass@4": 0.21916330239970003, + "pass@8": 0.22346821216105478, + "pass@16": 0.22872197474530756, + "pass@32": 0.23562179093658991, + "pass@64": 0.24510618631947784, + "pass@128": 0.258 + }, + "5": { + "pass@1": 0.2116171875, + "pass@2": 0.21723412893700783, + "pass@4": 0.22284044347581547, + "pass@8": 0.22918635588080552, + "pass@16": 0.23692081774417614, + "pass@32": 0.24659684355460448, + "pass@64": 0.25860912185373264, + "pass@128": 0.272 + }, + "6": { + "pass@1": 0.2580546875, + "pass@2": 0.262111343503937, + "pass@4": 0.26583126621672293, + "pass@8": 0.269561781409488, + "pass@16": 0.2735911486266416, + "pass@32": 0.2786441597934989, + "pass@64": 0.2852150182516317, + "pass@128": 0.293 + }, + "7": { + "pass@1": 0.195703125, + "pass@2": 0.2011721210629921, + "pass@4": 0.20649363198350204, + "pass@8": 0.212816902115117, + "pass@16": 0.22024960690112913, + "pass@32": 0.2283513294970211, + "pass@64": 0.23693507099206224, + "pass@128": 0.247 + }, + "8": { + "pass@1": 0.18925, + "pass@2": 0.19675135334645666, + "pass@4": 0.20417071484814395, + "pass@8": 0.21275267862195865, + "pass@16": 0.22278762964396398, + "pass@32": 0.23415911788441254, + "pass@64": 0.2463119989893093, + "pass@128": 0.258 + }, + "9": { + "pass@1": 0.175953125, + "pass@2": 0.18321788877952747, + "pass@4": 0.1904768334270717, + "pass@8": 0.19806054813384352, + "pass@16": 0.20626457792569045, + "pass@32": 0.21481318234518765, + "pass@64": 0.22277015534238243, + "pass@128": 0.231 + }, + "11": { + "pass@1": 0.1763515625, + "pass@2": 0.1854534940944882, + "pass@4": 0.1933735462129733, + "pass@8": 0.20147957136226124, + "pass@16": 0.21037463664286407, + "pass@32": 0.22037157099835816, + "pass@64": 0.23143147890828927, + "pass@128": 0.243 + }, + "12": { + "pass@1": 0.1750078125, + "pass@2": 0.18632972440944887, + "pass@4": 0.19674250149981243, + "pass@8": 0.2075677772984937, + "pass@16": 0.21899942949146833, + "pass@32": 0.23008881738333872, + "pass@64": 0.24004304392698875, + "pass@128": 0.249 + }, + "13": { + "pass@1": 0.14003125, + "pass@2": 0.15481865157480312, + "pass@4": 0.16692630361829766, + "pass@8": 0.17772499892830299, + "pass@16": 0.1882665689521189, + "pass@32": 0.19883793436883665, + "pass@64": 0.20967385171136876, + "pass@128": 0.221 + }, + "14": { + "pass@1": 0.1436328125, + "pass@2": 0.15869389763779526, + "pass@4": 0.17226924925009368, + "pass@8": 0.1845543174527869, + "pass@16": 0.19568671047542624, + "pass@32": 0.2065402438692653, + "pass@64": 0.21755881523205667, + "pass@128": 0.229 + }, + "15": { + "pass@1": 0.1338828125, + "pass@2": 0.14767212106299207, + "pass@4": 0.1592121984439444, + "pass@8": 0.1693664968394039, + "pass@16": 0.1790848548813016, + "pass@32": 0.1888553870136373, + "pass@64": 0.19929553877537814, + "pass@128": 0.211 + }, + "16": { + "pass@1": 0.155671875, + "pass@2": 0.16875172244094488, + "pass@4": 0.17969398078365192, + "pass@8": 0.1898988485117523, + "pass@16": 0.1991953499621563, + "pass@32": 0.20812386294309929, + "pass@64": 0.21791788802494516, + "pass@128": 0.23 + }, + "17": { + "pass@1": 0.14675, + "pass@2": 0.16014972933070862, + "pass@4": 0.16990317519685033, + "pass@8": 0.17901342366976633, + "pass@16": 0.1890970246223302, + "pass@32": 0.2009162762964177, + "pass@64": 0.21379618773979528, + "pass@128": 0.227 + }, + "18": { + "pass@1": 0.123859375, + "pass@2": 0.13845374015748027, + "pass@4": 0.15128099362579675, + "pass@8": 0.1636793233057228, + "pass@16": 0.17590328285323045, + "pass@32": 0.18769288846945548, + "pass@64": 0.19939121996448883, + "pass@128": 0.213 + }, + "19": { + "pass@1": 0.1425859375, + "pass@2": 0.1542038631889764, + "pass@4": 0.16308084739407575, + "pass@8": 0.17024050661671622, + "pass@16": 0.17678480815036582, + "pass@32": 0.18411748530607766, + "pass@64": 0.19279372965274186, + "pass@128": 0.203 + }, + "20": { + "pass@1": 0.12134375, + "pass@2": 0.13410039370078738, + "pass@4": 0.1453615757405324, + "pass@8": 0.1559681872763174, + "pass@16": 0.1660694483193041, + "pass@32": 0.17588726848327202, + "pass@64": 0.1855413942815293, + "pass@128": 0.196 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a234e1227d691a770ffd64d543f1d8acdce097c1 --- /dev/null +++ b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2", "results_dir": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs", "per_file": [{"file": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 9541, "extra_nodes": 1152000, "missing_nodes": 1152000, "perfect_with_extra": 0, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "extra_nodes": 1152000, "missing_nodes": 1152000, "answer_mismatch": 9541}}, "results_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 121879, "extra_nodes": 1279999, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "extra_nodes": 1279999, "missing_nodes": 1280000, "answer_mismatch": 121879}}, "results_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 131420, "extra_nodes": 2431999, "missing_nodes": 2432000, "perfect_with_extra": 0, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "extra_nodes": 2431999, "missing_nodes": 2432000, "answer_mismatch": 131420}}}} diff --git a/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/results.json b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..0eb8c3f6638c259ab3ae78163039fdb428d5b7af --- /dev/null +++ b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..43314955302f9422527ae188215db4322eb8dcf4 --- /dev/null +++ b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:968c7292d5ffa390a250d1268de665f37a960ee75282d094d9edc38122cbc9b8 +size 2616730794 diff --git a/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.csv b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9b900bc37db422c6196d324d39bb781ec904608d --- /dev/null +++ b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9917178819444444,0.90478203125,0.9459621710526316,0.0,0.0,0.0,180.0242638888889,346.5120359375,1152000,1280000,2432000 diff --git a/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.json b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..202e8a184ca59d14924713ec60c3f801726456c1 --- /dev/null +++ b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9917178819444444, + "ood_acc": 0.90478203125, + "total_acc": 0.9459621710526316, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 180.0242638888889, + "ood_avg_resp_len": 346.5120359375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2300580, + "answer_accuracy": 0.9459621710526316, + "avg_response_len": 267.64940707236843, + "resp_tokens_sum": 650923358, + "pass_at_k": { + "pass@1": 0.9459621710526316, + "pass@2": 0.965499624430164, + "pass@4": 0.9754853252816995, + "pass@8": 0.9815746569203379, + "pass@16": 0.9856429402662464, + "pass@32": 0.9886553766902775, + "pass@64": 0.9910604803835107, + "pass@128": 0.9930526315789474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9699453125, + "pass@2": 0.9797700541338586, + "pass@4": 0.9846088579865014, + "pass@8": 0.9883971297589192, + "pass@16": 0.991548693276063, + "pass@32": 0.9939062487742031, + "pass@64": 0.9956849154464417, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99759375, + "pass@2": 0.9991465305118111, + "pass@4": 0.9998152384701912, + "pass@8": 0.9999862821828273, + "pass@16": 0.9999999227084759, + "pass@32": 0.9999999999993933, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99834375, + "pass@2": 0.9994431594488189, + "pass@4": 0.9998681761342333, + "pass@8": 0.9999880140745279, + "pass@16": 0.9999999083912323, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933515625, + "pass@2": 0.9955663139763777, + "pass@4": 0.9974079118860144, + "pass@8": 0.9986958248958838, + "pass@16": 0.9994353865491737, + "pass@32": 0.9998235513802302, + "pass@64": 0.9999861740539754, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995625, + "pass@2": 0.99801156496063, + "pass@4": 0.9991621754780652, + "pass@8": 0.9997905041568963, + "pass@16": 0.9999826485802666, + "pass@32": 0.999999900717592, + "pass@64": 0.9999999999999175, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.991, + "pass@2": 0.9933371062992128, + "pass@4": 0.9947224583802023, + "pass@8": 0.9958695873510292, + "pass@16": 0.9966041585170369, + "pass@32": 0.9969280491355603, + "pass@64": 0.9969985543119962, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9948046875, + "pass@2": 0.99785125492126, + "pass@4": 0.998795172384702, + "pass@8": 0.9994316174246192, + "pass@16": 0.9998579688324685, + "pass@32": 0.9999931255781963, + "pass@64": 0.9999999975930302, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.984796875, + "pass@2": 0.9912790354330716, + "pass@4": 0.9942763324896889, + "pass@8": 0.996222904016301, + "pass@16": 0.9974501804727003, + "pass@32": 0.9981885625139278, + "pass@64": 0.998723119840615, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9512578125, + "pass@2": 0.9658013041338588, + "pass@4": 0.9731058290213717, + "pass@8": 0.9777503786832513, + "pass@16": 0.9812478554053372, + "pass@32": 0.9844936593333612, + "pass@64": 0.9875782128535292, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.9279609375, + "pass@2": 0.9452753444881897, + "pass@4": 0.9557707446569181, + "pass@8": 0.9629128336731604, + "pass@16": 0.9684001319561727, + "pass@32": 0.9734901999432878, + "pass@64": 0.9785937143414183, + "pass@128": 0.984 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9373233267716538, + "pass@4": 0.9503126651668536, + "pass@8": 0.9590150667585761, + "pass@16": 0.9658224358162538, + "pass@32": 0.9720045806193894, + "pass@64": 0.9776870791192475, + "pass@128": 0.983 + }, + "14": { + "pass@1": 0.9195390625, + "pass@2": 0.9461455462598434, + "pass@4": 0.9610489086989125, + "pass@8": 0.9705652801128191, + "pass@16": 0.976617495691548, + "pass@32": 0.9807539224865852, + "pass@64": 0.9838434201680727, + "pass@128": 0.986 + }, + "15": { + "pass@1": 0.9212421875, + "pass@2": 0.9496477608267725, + "pass@4": 0.9639471972253464, + "pass@8": 0.9731272495996897, + "pass@16": 0.9795135593649998, + "pass@32": 0.9843676266202894, + "pass@64": 0.9887092071015339, + "pass@128": 0.993 + }, + "16": { + "pass@1": 0.9186484375, + "pass@2": 0.9507684547244115, + "pass@4": 0.9659008573303339, + "pass@8": 0.974618462414321, + "pass@16": 0.9800429740397889, + "pass@32": 0.9841030640501913, + "pass@64": 0.9876616774077218, + "pass@128": 0.99 + }, + "17": { + "pass@1": 0.9137890625, + "pass@2": 0.9511459153543321, + "pass@4": 0.9683324656917881, + "pass@8": 0.9782519917436393, + "pass@16": 0.9845447476466959, + "pass@32": 0.9884888034847372, + "pass@64": 0.9909003607208057, + "pass@128": 0.993 + }, + "18": { + "pass@1": 0.8819921875, + "pass@2": 0.9288923474409468, + "pass@4": 0.9518024659730036, + "pass@8": 0.9643686704288694, + "pass@16": 0.9725036453284903, + "pass@32": 0.9785188430691222, + "pass@64": 0.9827390755534815, + "pass@128": 0.985 + }, + "19": { + "pass@1": 0.8668359375, + "pass@2": 0.9196662155511819, + "pass@4": 0.9460197001312345, + "pass@8": 0.9618559891779669, + "pass@16": 0.9724486989570319, + "pass@32": 0.9796740074314165, + "pass@64": 0.9850677359090562, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.831703125, + "pass@2": 0.8954216289370094, + "pass@4": 0.9293240232470946, + "pass@8": 0.9490706950329557, + "pass@16": 0.9611954535250025, + "pass@32": 0.9697180119787673, + "pass@64": 0.9759758828658656, + "pass@128": 0.981 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f754a36a0831e027c84c0f1abe136faa58465600 --- /dev/null +++ b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2", "results_dir": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs", "per_file": [{"file": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 2173, "dependency_mismatch": 933959, "answer_mismatch": 8758, "extra_nodes": 1151956, "missing_nodes": 1152000, "perfect_with_extra": 0, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "dependency_mismatch": 933959, "extra_nodes": 1151956, "missing_nodes": 1152000, "answer_mismatch": 8758, "value_mismatch": 2173}}, "results_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 46724, "dependency_mismatch": 1062723, "answer_mismatch": 111672, "extra_nodes": 1279965, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "dependency_mismatch": 1062723, "answer_mismatch": 111672, "extra_nodes": 1279965, "missing_nodes": 1280000, "value_mismatch": 46724}}, "results_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 48897, "dependency_mismatch": 1996682, "answer_mismatch": 120430, "extra_nodes": 2431921, "missing_nodes": 2432000, "perfect_with_extra": 0, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "dependency_mismatch": 1996682, "extra_nodes": 2431921, "missing_nodes": 2432000, "answer_mismatch": 120430, "value_mismatch": 48897}}}} diff --git a/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/results.json b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..0eb8c3f6638c259ab3ae78163039fdb428d5b7af --- /dev/null +++ b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a31776f6f18efdef0bff2dad23b5c454ed81f689 --- /dev/null +++ b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb59d98db815792924ef404ed5ac8271c437ee058e414b57938353dcb1adef5d +size 1674664484 diff --git a/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_metrics.json b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3179173361e373e0cb2ce41363c53d3427bee7ef --- /dev/null +++ b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9923975694444445, + "avg_response_len": 176.4994105902778, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9923975694444445, + "pass@2": 0.9957820018591392, + "pass@4": 0.9971995788963887, + "pass@8": 0.9979632664900273, + "pass@16": 0.9984037349725341, + "pass@32": 0.9987061615186326, + "pass@64": 0.9988642449684446, + "pass@128": 0.9988888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9722734375, + "pass@2": 0.982090920275591, + "pass@4": 0.9870395954255715, + "pass@8": 0.9898463912960546, + "pass@16": 0.9913574932933285, + "pass@32": 0.9924062358073099, + "pass@64": 0.9929435009525102, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99696875, + "pass@2": 0.9987176427165356, + "pass@4": 0.999073522215973, + "pass@8": 0.9991772006158419, + "pass@16": 0.9993323490768988, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998421875, + "pass@2": 0.999459153543307, + "pass@4": 0.9998440072178476, + "pass@8": 0.999979701409027, + "pass@16": 0.9999997111918157, + "pass@32": 0.9999999999851739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941015625, + "pass@2": 0.9965524114173228, + "pass@4": 0.9978783887326584, + "pass@8": 0.9987054335254999, + "pass@16": 0.9993742678247356, + "pass@32": 0.9998140401510567, + "pass@64": 0.9999861487158879, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99475, + "pass@2": 0.9969265501968505, + "pass@4": 0.9980495649606299, + "pass@8": 0.9987365127580474, + "pass@16": 0.9989781223272445, + "pass@32": 0.9989999037739257, + "pass@64": 0.9989999999999666, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9951015625, + "pass@2": 0.9976016240157483, + "pass@4": 0.9984762033183351, + "pass@8": 0.9988805544747335, + "pass@16": 0.9989943223963219, + "pass@32": 0.9989999945303923, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.992984375, + "pass@2": 0.9974980314960634, + "pass@4": 0.9988455806149232, + "pass@8": 0.9992881325051802, + "pass@16": 0.9995580380881292, + "pass@32": 0.999829071592898, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9869765625, + "pass@2": 0.9931916830708665, + "pass@4": 0.995589347581552, + "pass@8": 0.9970554718258637, + "pass@16": 0.9980393105543356, + "pass@32": 0.9987247492592555, + "pass@64": 0.998984425990599, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9722734375, + "2": 1.0, + "3": 0.99696875, + "4": 0.998421875, + "5": 0.9941015625, + "6": 0.99475, + "7": 0.9951015625, + "8": 0.992984375, + "9": 0.9869765625 + }, + "per_op_avg_response_len": { + "10": 245.9643203125, + "2": 103.040046875, + "3": 121.6964375, + "4": 141.7835859375, + "5": 164.8098203125, + "6": 175.4171015625, + "7": 190.5570234375, + "8": 213.8619296875, + "9": 231.3644296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 245.9643203125, + "2": 103.040046875, + "3": 121.6964375, + "4": 141.7835859375, + "5": 164.8098203125, + "6": 175.4171015625, + "7": 190.5570234375, + "8": 213.8619296875, + "9": 231.3644296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 1143242, + "answer_accuracy": 0.9923975694444445, + "avg_response_len": 176.4994105902778, + "resp_tokens_sum": 203327321, + "pass_at_k": { + "pass@1": 0.9923975694444445, + "pass@2": 0.9957820018591392, + "pass@4": 0.9971995788963887, + "pass@8": 0.9979632664900273, + "pass@16": 0.9984037349725341, + "pass@32": 0.9987061615186326, + "pass@64": 0.9988642449684446, + "pass@128": 0.9988888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9722734375, + "pass@2": 0.982090920275591, + "pass@4": 0.9870395954255715, + "pass@8": 0.9898463912960546, + "pass@16": 0.9913574932933285, + "pass@32": 0.9924062358073099, + "pass@64": 0.9929435009525102, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99696875, + "pass@2": 0.9987176427165356, + "pass@4": 0.999073522215973, + "pass@8": 0.9991772006158419, + "pass@16": 0.9993323490768988, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998421875, + "pass@2": 0.999459153543307, + "pass@4": 0.9998440072178476, + "pass@8": 0.999979701409027, + "pass@16": 0.9999997111918157, + "pass@32": 0.9999999999851739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941015625, + "pass@2": 0.9965524114173228, + "pass@4": 0.9978783887326584, + "pass@8": 0.9987054335254999, + "pass@16": 0.9993742678247356, + "pass@32": 0.9998140401510567, + "pass@64": 0.9999861487158879, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99475, + "pass@2": 0.9969265501968505, + "pass@4": 0.9980495649606299, + "pass@8": 0.9987365127580474, + "pass@16": 0.9989781223272445, + "pass@32": 0.9989999037739257, + "pass@64": 0.9989999999999666, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9951015625, + "pass@2": 0.9976016240157483, + "pass@4": 0.9984762033183351, + "pass@8": 0.9988805544747335, + "pass@16": 0.9989943223963219, + "pass@32": 0.9989999945303923, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.992984375, + "pass@2": 0.9974980314960634, + "pass@4": 0.9988455806149232, + "pass@8": 0.9992881325051802, + "pass@16": 0.9995580380881292, + "pass@32": 0.999829071592898, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9869765625, + "pass@2": 0.9931916830708665, + "pass@4": 0.995589347581552, + "pass@8": 0.9970554718258637, + "pass@16": 0.9980393105543356, + "pass@32": 0.9987247492592555, + "pass@64": 0.998984425990599, + "pass@128": 0.999 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.91275625, + "avg_response_len": 339.53681640625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.91275625, + "pass@2": 0.9458462967519363, + "pass@4": 0.9624475184758139, + "pass@8": 0.9726641426761587, + "pass@16": 0.9796753897332539, + "pass@32": 0.9849077514114102, + "pass@64": 0.988984272764239, + "pass@128": 0.9919 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9569375, + "pass@2": 0.9694979084645677, + "pass@4": 0.9755786563554552, + "pass@8": 0.9795771285286979, + "pass@16": 0.9828716597217099, + "pass@32": 0.9858994698672985, + "pass@64": 0.9886172988906358, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9321875, + "pass@2": 0.9527299458661426, + "pass@4": 0.964783362579677, + "pass@8": 0.9725947365272651, + "pass@16": 0.9780288578767546, + "pass@32": 0.9822927738860049, + "pass@64": 0.9860262163530781, + "pass@128": 0.989 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9376562500000003, + "pass@4": 0.9520333217097862, + "pass@8": 0.9627994330286813, + "pass@16": 0.9713319331348897, + "pass@32": 0.9782062595316485, + "pass@64": 0.9837086690994824, + "pass@128": 0.988 + }, + "14": { + "pass@1": 0.92659375, + "pass@2": 0.9529224901574813, + "pass@4": 0.9665270294338203, + "pass@8": 0.9751033296649382, + "pass@16": 0.9814160863652701, + "pass@32": 0.9866717919871326, + "pass@64": 0.99098669283729, + "pass@128": 0.993 + }, + "15": { + "pass@1": 0.9321484375, + "pass@2": 0.9586374261811037, + "pass@4": 0.9716738764529432, + "pass@8": 0.980257322244576, + "pass@16": 0.9860876075856799, + "pass@32": 0.9905182347017089, + "pass@64": 0.9941053539635863, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.919421875, + "pass@2": 0.9510446604330718, + "pass@4": 0.9672112351893514, + "pass@8": 0.9771642191649528, + "pass@16": 0.9837104050806683, + "pass@32": 0.9884127699664014, + "pass@64": 0.9923083437843704, + "pass@128": 0.995 + }, + "17": { + "pass@1": 0.917234375, + "pass@2": 0.9551825787401591, + "pass@4": 0.9728983112110978, + "pass@8": 0.9831834738064922, + "pass@16": 0.989503100920689, + "pass@32": 0.9936977352294757, + "pass@64": 0.9968179233919849, + "pass@128": 0.999 + }, + "18": { + "pass@1": 0.8954921875, + "pass@2": 0.9377255167322847, + "pass@4": 0.9575529777840263, + "pass@8": 0.9690916457538964, + "pass@16": 0.9765775044869797, + "pass@32": 0.9819170635269723, + "pass@64": 0.9859361791907656, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.875109375, + "pass@2": 0.9267666092519707, + "pass@4": 0.9520276386389189, + "pass@8": 0.9663416476690758, + "pass@16": 0.975462333146504, + "pass@32": 0.981573341392933, + "pass@64": 0.9858043859763082, + "pass@128": 0.989 + }, + "20": { + "pass@1": 0.8575859375, + "pass@2": 0.9162995816929158, + "pass@4": 0.944188775403074, + "pass@8": 0.9605284903730037, + "pass@16": 0.9717644090133872, + "pass@32": 0.9798880740245233, + "pass@64": 0.9855316641548822, + "pass@128": 0.989 + } + }, + "per_op_accuracy": { + "11": 0.9569375, + "12": 0.9321875, + "13": 0.9148515625, + "14": 0.92659375, + "15": 0.9321484375, + "16": 0.919421875, + "17": 0.917234375, + "18": 0.8954921875, + "19": 0.875109375, + "20": 0.8575859375 + }, + "per_op_avg_response_len": { + "11": 266.557984375, + "12": 278.4357265625, + "13": 288.8397265625, + "14": 317.2391484375, + "15": 331.4378515625, + "16": 357.8768828125, + "17": 368.6404765625, + "18": 382.5142890625, + "19": 393.4576171875, + "20": 410.3684609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 266.557984375, + "12": 278.4357265625, + "13": 288.8397265625, + "14": 317.2391484375, + "15": 331.4378515625, + "16": 357.8768828125, + "17": 368.6404765625, + "18": 382.5142890625, + "19": 393.4576171875, + "20": 410.3684609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 1168328, + "answer_accuracy": 0.91275625, + "avg_response_len": 339.53681640625, + "resp_tokens_sum": 434607125, + "pass_at_k": { + "pass@1": 0.91275625, + "pass@2": 0.9458462967519363, + "pass@4": 0.9624475184758139, + "pass@8": 0.9726641426761587, + "pass@16": 0.9796753897332539, + "pass@32": 0.9849077514114102, + "pass@64": 0.988984272764239, + "pass@128": 0.9919 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9569375, + "pass@2": 0.9694979084645677, + "pass@4": 0.9755786563554552, + "pass@8": 0.9795771285286979, + "pass@16": 0.9828716597217099, + "pass@32": 0.9858994698672985, + "pass@64": 0.9886172988906358, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9321875, + "pass@2": 0.9527299458661426, + "pass@4": 0.964783362579677, + "pass@8": 0.9725947365272651, + "pass@16": 0.9780288578767546, + "pass@32": 0.9822927738860049, + "pass@64": 0.9860262163530781, + "pass@128": 0.989 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9376562500000003, + "pass@4": 0.9520333217097862, + "pass@8": 0.9627994330286813, + "pass@16": 0.9713319331348897, + "pass@32": 0.9782062595316485, + "pass@64": 0.9837086690994824, + "pass@128": 0.988 + }, + "14": { + "pass@1": 0.92659375, + "pass@2": 0.9529224901574813, + "pass@4": 0.9665270294338203, + "pass@8": 0.9751033296649382, + "pass@16": 0.9814160863652701, + "pass@32": 0.9866717919871326, + "pass@64": 0.99098669283729, + "pass@128": 0.993 + }, + "15": { + "pass@1": 0.9321484375, + "pass@2": 0.9586374261811037, + "pass@4": 0.9716738764529432, + "pass@8": 0.980257322244576, + "pass@16": 0.9860876075856799, + "pass@32": 0.9905182347017089, + "pass@64": 0.9941053539635863, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.919421875, + "pass@2": 0.9510446604330718, + "pass@4": 0.9672112351893514, + "pass@8": 0.9771642191649528, + "pass@16": 0.9837104050806683, + "pass@32": 0.9884127699664014, + "pass@64": 0.9923083437843704, + "pass@128": 0.995 + }, + "17": { + "pass@1": 0.917234375, + "pass@2": 0.9551825787401591, + "pass@4": 0.9728983112110978, + "pass@8": 0.9831834738064922, + "pass@16": 0.989503100920689, + "pass@32": 0.9936977352294757, + "pass@64": 0.9968179233919849, + "pass@128": 0.999 + }, + "18": { + "pass@1": 0.8954921875, + "pass@2": 0.9377255167322847, + "pass@4": 0.9575529777840263, + "pass@8": 0.9690916457538964, + "pass@16": 0.9765775044869797, + "pass@32": 0.9819170635269723, + "pass@64": 0.9859361791907656, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.875109375, + "pass@2": 0.9267666092519707, + "pass@4": 0.9520276386389189, + "pass@8": 0.9663416476690758, + "pass@16": 0.975462333146504, + "pass@32": 0.981573341392933, + "pass@64": 0.9858043859763082, + "pass@128": 0.989 + }, + "20": { + "pass@1": 0.8575859375, + "pass@2": 0.9162995816929158, + "pass@4": 0.944188775403074, + "pass@8": 0.9605284903730037, + "pass@16": 0.9717644090133872, + "pass@32": 0.9798880740245233, + "pass@64": 0.9855316641548822, + "pass@128": 0.989 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.9504810855263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9504810855263158, + "pass@2": 0.9695000518027258, + "pass@4": 0.9789090207802891, + "pass@8": 0.9846479381669422, + "pass@16": 0.988546711162386, + "pass@32": 0.9914438404095707, + "pass@64": 0.9936642595978096, + "pass@128": 0.9952105263157894 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9722734375, + "pass@2": 0.982090920275591, + "pass@4": 0.9870395954255715, + "pass@8": 0.9898463912960546, + "pass@16": 0.9913574932933285, + "pass@32": 0.9924062358073099, + "pass@64": 0.9929435009525102, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99696875, + "pass@2": 0.9987176427165356, + "pass@4": 0.999073522215973, + "pass@8": 0.9991772006158419, + "pass@16": 0.9993323490768988, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998421875, + "pass@2": 0.999459153543307, + "pass@4": 0.9998440072178476, + "pass@8": 0.999979701409027, + "pass@16": 0.9999997111918157, + "pass@32": 0.9999999999851739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941015625, + "pass@2": 0.9965524114173228, + "pass@4": 0.9978783887326584, + "pass@8": 0.9987054335254999, + "pass@16": 0.9993742678247356, + "pass@32": 0.9998140401510567, + "pass@64": 0.9999861487158879, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99475, + "pass@2": 0.9969265501968505, + "pass@4": 0.9980495649606299, + "pass@8": 0.9987365127580474, + "pass@16": 0.9989781223272445, + "pass@32": 0.9989999037739257, + "pass@64": 0.9989999999999666, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9951015625, + "pass@2": 0.9976016240157483, + "pass@4": 0.9984762033183351, + "pass@8": 0.9988805544747335, + "pass@16": 0.9989943223963219, + "pass@32": 0.9989999945303923, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.992984375, + "pass@2": 0.9974980314960634, + "pass@4": 0.9988455806149232, + "pass@8": 0.9992881325051802, + "pass@16": 0.9995580380881292, + "pass@32": 0.999829071592898, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9869765625, + "pass@2": 0.9931916830708665, + "pass@4": 0.995589347581552, + "pass@8": 0.9970554718258637, + "pass@16": 0.9980393105543356, + "pass@32": 0.9987247492592555, + "pass@64": 0.998984425990599, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9569375, + "pass@2": 0.9694979084645677, + "pass@4": 0.9755786563554552, + "pass@8": 0.9795771285286979, + "pass@16": 0.9828716597217099, + "pass@32": 0.9858994698672985, + "pass@64": 0.9886172988906358, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9321875, + "pass@2": 0.9527299458661426, + "pass@4": 0.964783362579677, + "pass@8": 0.9725947365272651, + "pass@16": 0.9780288578767546, + "pass@32": 0.9822927738860049, + "pass@64": 0.9860262163530781, + "pass@128": 0.989 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9376562500000003, + "pass@4": 0.9520333217097862, + "pass@8": 0.9627994330286813, + "pass@16": 0.9713319331348897, + "pass@32": 0.9782062595316485, + "pass@64": 0.9837086690994824, + "pass@128": 0.988 + }, + "14": { + "pass@1": 0.92659375, + "pass@2": 0.9529224901574813, + "pass@4": 0.9665270294338203, + "pass@8": 0.9751033296649382, + "pass@16": 0.9814160863652701, + "pass@32": 0.9866717919871326, + "pass@64": 0.99098669283729, + "pass@128": 0.993 + }, + "15": { + "pass@1": 0.9321484375, + "pass@2": 0.9586374261811037, + "pass@4": 0.9716738764529432, + "pass@8": 0.980257322244576, + "pass@16": 0.9860876075856799, + "pass@32": 0.9905182347017089, + "pass@64": 0.9941053539635863, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.919421875, + "pass@2": 0.9510446604330718, + "pass@4": 0.9672112351893514, + "pass@8": 0.9771642191649528, + "pass@16": 0.9837104050806683, + "pass@32": 0.9884127699664014, + "pass@64": 0.9923083437843704, + "pass@128": 0.995 + }, + "17": { + "pass@1": 0.917234375, + "pass@2": 0.9551825787401591, + "pass@4": 0.9728983112110978, + "pass@8": 0.9831834738064922, + "pass@16": 0.989503100920689, + "pass@32": 0.9936977352294757, + "pass@64": 0.9968179233919849, + "pass@128": 0.999 + }, + "18": { + "pass@1": 0.8954921875, + "pass@2": 0.9377255167322847, + "pass@4": 0.9575529777840263, + "pass@8": 0.9690916457538964, + "pass@16": 0.9765775044869797, + "pass@32": 0.9819170635269723, + "pass@64": 0.9859361791907656, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.875109375, + "pass@2": 0.9267666092519707, + "pass@4": 0.9520276386389189, + "pass@8": 0.9663416476690758, + "pass@16": 0.975462333146504, + "pass@32": 0.981573341392933, + "pass@64": 0.9858043859763082, + "pass@128": 0.989 + }, + "20": { + "pass@1": 0.8575859375, + "pass@2": 0.9162995816929158, + "pass@4": 0.944188775403074, + "pass@8": 0.9605284903730037, + "pass@16": 0.9717644090133872, + "pass@32": 0.9798880740245233, + "pass@64": 0.9855316641548822, + "pass@128": 0.989 + } + }, + "per_op_accuracy": { + "10": 0.9722734375, + "2": 1.0, + "3": 0.99696875, + "4": 0.998421875, + "5": 0.9941015625, + "6": 0.99475, + "7": 0.9951015625, + "8": 0.992984375, + "9": 0.9869765625, + "11": 0.9569375, + "12": 0.9321875, + "13": 0.9148515625, + "14": 0.92659375, + "15": 0.9321484375, + "16": 0.919421875, + "17": 0.917234375, + "18": 0.8954921875, + "19": 0.875109375, + "20": 0.8575859375 + }, + "per_op_avg_response_len": { + "10": 245.9643203125, + "2": 103.040046875, + "3": 121.6964375, + "4": 141.7835859375, + "5": 164.8098203125, + "6": 175.4171015625, + "7": 190.5570234375, + "8": 213.8619296875, + "9": 231.3644296875, + "11": 266.557984375, + "12": 278.4357265625, + "13": 288.8397265625, + "14": 317.2391484375, + "15": 331.4378515625, + "16": 357.8768828125, + "17": 368.6404765625, + "18": 382.5142890625, + "19": 393.4576171875, + "20": 410.3684609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 245.9643203125, + "2": 103.040046875, + "3": 121.6964375, + "4": 141.7835859375, + "5": 164.8098203125, + "6": 175.4171015625, + "7": 190.5570234375, + "8": 213.8619296875, + "9": 231.3644296875, + "11": 266.557984375, + "12": 278.4357265625, + "13": 288.8397265625, + "14": 317.2391484375, + "15": 331.4378515625, + "16": 357.8768828125, + "17": 368.6404765625, + "18": 382.5142890625, + "19": 393.4576171875, + "20": 410.3684609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2311570, + "answer_accuracy": 0.9504810855263158, + "avg_response_len": 262.30857154605263, + "resp_tokens_sum": 637934446, + "pass_at_k": { + "pass@1": 0.9504810855263158, + "pass@2": 0.9695000518027258, + "pass@4": 0.9789090207802891, + "pass@8": 0.9846479381669422, + "pass@16": 0.988546711162386, + "pass@32": 0.9914438404095707, + "pass@64": 0.9936642595978096, + "pass@128": 0.9952105263157894 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9722734375, + "pass@2": 0.982090920275591, + "pass@4": 0.9870395954255715, + "pass@8": 0.9898463912960546, + "pass@16": 0.9913574932933285, + "pass@32": 0.9924062358073099, + "pass@64": 0.9929435009525102, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99696875, + "pass@2": 0.9987176427165356, + "pass@4": 0.999073522215973, + "pass@8": 0.9991772006158419, + "pass@16": 0.9993323490768988, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998421875, + "pass@2": 0.999459153543307, + "pass@4": 0.9998440072178476, + "pass@8": 0.999979701409027, + "pass@16": 0.9999997111918157, + "pass@32": 0.9999999999851739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941015625, + "pass@2": 0.9965524114173228, + "pass@4": 0.9978783887326584, + "pass@8": 0.9987054335254999, + "pass@16": 0.9993742678247356, + "pass@32": 0.9998140401510567, + "pass@64": 0.9999861487158879, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99475, + "pass@2": 0.9969265501968505, + "pass@4": 0.9980495649606299, + "pass@8": 0.9987365127580474, + "pass@16": 0.9989781223272445, + "pass@32": 0.9989999037739257, + "pass@64": 0.9989999999999666, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9951015625, + "pass@2": 0.9976016240157483, + "pass@4": 0.9984762033183351, + "pass@8": 0.9988805544747335, + "pass@16": 0.9989943223963219, + "pass@32": 0.9989999945303923, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.992984375, + "pass@2": 0.9974980314960634, + "pass@4": 0.9988455806149232, + "pass@8": 0.9992881325051802, + "pass@16": 0.9995580380881292, + "pass@32": 0.999829071592898, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9869765625, + "pass@2": 0.9931916830708665, + "pass@4": 0.995589347581552, + "pass@8": 0.9970554718258637, + "pass@16": 0.9980393105543356, + "pass@32": 0.9987247492592555, + "pass@64": 0.998984425990599, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9569375, + "pass@2": 0.9694979084645677, + "pass@4": 0.9755786563554552, + "pass@8": 0.9795771285286979, + "pass@16": 0.9828716597217099, + "pass@32": 0.9858994698672985, + "pass@64": 0.9886172988906358, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9321875, + "pass@2": 0.9527299458661426, + "pass@4": 0.964783362579677, + "pass@8": 0.9725947365272651, + "pass@16": 0.9780288578767546, + "pass@32": 0.9822927738860049, + "pass@64": 0.9860262163530781, + "pass@128": 0.989 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9376562500000003, + "pass@4": 0.9520333217097862, + "pass@8": 0.9627994330286813, + "pass@16": 0.9713319331348897, + "pass@32": 0.9782062595316485, + "pass@64": 0.9837086690994824, + "pass@128": 0.988 + }, + "14": { + "pass@1": 0.92659375, + "pass@2": 0.9529224901574813, + "pass@4": 0.9665270294338203, + "pass@8": 0.9751033296649382, + "pass@16": 0.9814160863652701, + "pass@32": 0.9866717919871326, + "pass@64": 0.99098669283729, + "pass@128": 0.993 + }, + "15": { + "pass@1": 0.9321484375, + "pass@2": 0.9586374261811037, + "pass@4": 0.9716738764529432, + "pass@8": 0.980257322244576, + "pass@16": 0.9860876075856799, + "pass@32": 0.9905182347017089, + "pass@64": 0.9941053539635863, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.919421875, + "pass@2": 0.9510446604330718, + "pass@4": 0.9672112351893514, + "pass@8": 0.9771642191649528, + "pass@16": 0.9837104050806683, + "pass@32": 0.9884127699664014, + "pass@64": 0.9923083437843704, + "pass@128": 0.995 + }, + "17": { + "pass@1": 0.917234375, + "pass@2": 0.9551825787401591, + "pass@4": 0.9728983112110978, + "pass@8": 0.9831834738064922, + "pass@16": 0.989503100920689, + "pass@32": 0.9936977352294757, + "pass@64": 0.9968179233919849, + "pass@128": 0.999 + }, + "18": { + "pass@1": 0.8954921875, + "pass@2": 0.9377255167322847, + "pass@4": 0.9575529777840263, + "pass@8": 0.9690916457538964, + "pass@16": 0.9765775044869797, + "pass@32": 0.9819170635269723, + "pass@64": 0.9859361791907656, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.875109375, + "pass@2": 0.9267666092519707, + "pass@4": 0.9520276386389189, + "pass@8": 0.9663416476690758, + "pass@16": 0.975462333146504, + "pass@32": 0.981573341392933, + "pass@64": 0.9858043859763082, + "pass@128": 0.989 + }, + "20": { + "pass@1": 0.8575859375, + "pass@2": 0.9162995816929158, + "pass@4": 0.944188775403074, + "pass@8": 0.9605284903730037, + "pass@16": 0.9717644090133872, + "pass@32": 0.9798880740245233, + "pass@64": 0.9855316641548822, + "pass@128": 0.989 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a4b3811991996cf9cd7e72134fa3b117e6b1eb02 --- /dev/null +++ b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf1178ba74866a661cdbd18ff34a89792dd569d167d8962551e823a762f7bd5 +size 2618033145 diff --git a/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.csv b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a2e1e3f02b44aa674dab9808eed0e7b68777ef45 --- /dev/null +++ b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9923975694444445,0.91275625,0.9504810855263158,0.0,0.0,0.0,176.4994105902778,339.53681640625,1152000,1280000,2432000 diff --git a/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.json b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a0d7ae7443229df3dbe657423907ea1690fe483b --- /dev/null +++ b/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9923975694444445, + "ood_acc": 0.91275625, + "total_acc": 0.9504810855263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 176.4994105902778, + "ood_avg_resp_len": 339.53681640625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2311570, + "answer_accuracy": 0.9504810855263158, + "avg_response_len": 262.30857154605263, + "resp_tokens_sum": 637934446, + "pass_at_k": { + "pass@1": 0.9504810855263158, + "pass@2": 0.9695000518027258, + "pass@4": 0.9789090207802891, + "pass@8": 0.9846479381669422, + "pass@16": 0.988546711162386, + "pass@32": 0.9914438404095707, + "pass@64": 0.9936642595978096, + "pass@128": 0.9952105263157894 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9722734375, + "pass@2": 0.982090920275591, + "pass@4": 0.9870395954255715, + "pass@8": 0.9898463912960546, + "pass@16": 0.9913574932933285, + "pass@32": 0.9924062358073099, + "pass@64": 0.9929435009525102, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99696875, + "pass@2": 0.9987176427165356, + "pass@4": 0.999073522215973, + "pass@8": 0.9991772006158419, + "pass@16": 0.9993323490768988, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998421875, + "pass@2": 0.999459153543307, + "pass@4": 0.9998440072178476, + "pass@8": 0.999979701409027, + "pass@16": 0.9999997111918157, + "pass@32": 0.9999999999851739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941015625, + "pass@2": 0.9965524114173228, + "pass@4": 0.9978783887326584, + "pass@8": 0.9987054335254999, + "pass@16": 0.9993742678247356, + "pass@32": 0.9998140401510567, + "pass@64": 0.9999861487158879, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99475, + "pass@2": 0.9969265501968505, + "pass@4": 0.9980495649606299, + "pass@8": 0.9987365127580474, + "pass@16": 0.9989781223272445, + "pass@32": 0.9989999037739257, + "pass@64": 0.9989999999999666, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9951015625, + "pass@2": 0.9976016240157483, + "pass@4": 0.9984762033183351, + "pass@8": 0.9988805544747335, + "pass@16": 0.9989943223963219, + "pass@32": 0.9989999945303923, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.992984375, + "pass@2": 0.9974980314960634, + "pass@4": 0.9988455806149232, + "pass@8": 0.9992881325051802, + "pass@16": 0.9995580380881292, + "pass@32": 0.999829071592898, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9869765625, + "pass@2": 0.9931916830708665, + "pass@4": 0.995589347581552, + "pass@8": 0.9970554718258637, + "pass@16": 0.9980393105543356, + "pass@32": 0.9987247492592555, + "pass@64": 0.998984425990599, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9569375, + "pass@2": 0.9694979084645677, + "pass@4": 0.9755786563554552, + "pass@8": 0.9795771285286979, + "pass@16": 0.9828716597217099, + "pass@32": 0.9858994698672985, + "pass@64": 0.9886172988906358, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9321875, + "pass@2": 0.9527299458661426, + "pass@4": 0.964783362579677, + "pass@8": 0.9725947365272651, + "pass@16": 0.9780288578767546, + "pass@32": 0.9822927738860049, + "pass@64": 0.9860262163530781, + "pass@128": 0.989 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9376562500000003, + "pass@4": 0.9520333217097862, + "pass@8": 0.9627994330286813, + "pass@16": 0.9713319331348897, + "pass@32": 0.9782062595316485, + "pass@64": 0.9837086690994824, + "pass@128": 0.988 + }, + "14": { + "pass@1": 0.92659375, + "pass@2": 0.9529224901574813, + "pass@4": 0.9665270294338203, + "pass@8": 0.9751033296649382, + "pass@16": 0.9814160863652701, + "pass@32": 0.9866717919871326, + "pass@64": 0.99098669283729, + "pass@128": 0.993 + }, + "15": { + "pass@1": 0.9321484375, + "pass@2": 0.9586374261811037, + "pass@4": 0.9716738764529432, + "pass@8": 0.980257322244576, + "pass@16": 0.9860876075856799, + "pass@32": 0.9905182347017089, + "pass@64": 0.9941053539635863, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.919421875, + "pass@2": 0.9510446604330718, + "pass@4": 0.9672112351893514, + "pass@8": 0.9771642191649528, + "pass@16": 0.9837104050806683, + "pass@32": 0.9884127699664014, + "pass@64": 0.9923083437843704, + "pass@128": 0.995 + }, + "17": { + "pass@1": 0.917234375, + "pass@2": 0.9551825787401591, + "pass@4": 0.9728983112110978, + "pass@8": 0.9831834738064922, + "pass@16": 0.989503100920689, + "pass@32": 0.9936977352294757, + "pass@64": 0.9968179233919849, + "pass@128": 0.999 + }, + "18": { + "pass@1": 0.8954921875, + "pass@2": 0.9377255167322847, + "pass@4": 0.9575529777840263, + "pass@8": 0.9690916457538964, + "pass@16": 0.9765775044869797, + "pass@32": 0.9819170635269723, + "pass@64": 0.9859361791907656, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.875109375, + "pass@2": 0.9267666092519707, + "pass@4": 0.9520276386389189, + "pass@8": 0.9663416476690758, + "pass@16": 0.975462333146504, + "pass@32": 0.981573341392933, + "pass@64": 0.9858043859763082, + "pass@128": 0.989 + }, + "20": { + "pass@1": 0.8575859375, + "pass@2": 0.9162995816929158, + "pass@4": 0.944188775403074, + "pass@8": 0.9605284903730037, + "pass@16": 0.9717644090133872, + "pass@32": 0.9798880740245233, + "pass@64": 0.9855316641548822, + "pass@128": 0.989 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl b/Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c9ef5c78344da1155fcd6168b92651725ff10a19 --- /dev/null +++ b/Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fa35f6d2d084c259ecd5f0a8f1e745e8bb7ac8fbd684e1a93afc6cbc6a308b5 +size 1789209154 diff --git a/Qwen3-100M-ep4/composition/checkpoint-85276_metrics.json b/Qwen3-100M-ep4/composition/checkpoint-85276_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1f1fffa850ea1d47ba519b03ac38006425bd4c74 --- /dev/null +++ b/Qwen3-100M-ep4/composition/checkpoint-85276_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-85276", + "id": { + "answer_accuracy": 0.9949305555555555, + "avg_response_len": 200.0108559027778, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9949305555555555, + "pass@2": 0.9962390912073487, + "pass@4": 0.9971986516685417, + "pass@8": 0.9977962149213019, + "pass@16": 0.9981298764933108, + "pass@32": 0.9983723358418248, + "pass@64": 0.9986656367264329, + "pass@128": 0.9991111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9624921875, + "pass@2": 0.9715636072834645, + "pass@4": 0.9781628562054747, + "pass@8": 0.9820358637737919, + "pass@16": 0.984177149756922, + "pass@32": 0.9859330858922759, + "pass@64": 0.988238824249548, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999453125, + "pass@2": 0.9997881397637796, + "pass@4": 0.9999573371766028, + "pass@8": 0.9999984489981919, + "pass@16": 0.9999999988246276, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998984375, + "pass@2": 0.9999944635826771, + "pass@4": 0.9999999803149605, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.996578125, + "pass@2": 0.9977417568897637, + "pass@4": 0.998505216816648, + "pass@8": 0.9989649390790856, + "pass@16": 0.9992179855257273, + "pass@32": 0.9994388347222112, + "pass@64": 0.9997519685037116, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959609375, + "pass@2": 0.9970638533464568, + "pass@4": 0.9981624745031872, + "pass@8": 0.9991666824406795, + "pass@16": 0.9997737543325198, + "pass@32": 0.999979101961939, + "pass@64": 0.9999999377846387, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9624921875, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999453125, + "6": 0.9999921875, + "7": 0.9998984375, + "8": 0.996578125, + "9": 0.9959609375 + }, + "per_op_avg_response_len": { + "10": 281.153859375, + "2": 115.629984375, + "3": 144.724578125, + "4": 159.8816328125, + "5": 172.73553125, + "6": 198.9245234375, + "7": 223.9551328125, + "8": 243.5811875, + "9": 259.5112734375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.153859375, + "2": 115.629984375, + "3": 144.724578125, + "4": 159.8816328125, + "5": 172.73553125, + "6": 198.9245234375, + "7": 223.9551328125, + "8": 243.5811875, + "9": 259.5112734375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377286, + "answer_accuracy": 0.9954565602836879, + "avg_response_len": 190.53283571850727, + "resp_tokens_sum": 72213469, + "pass_at_k": { + "pass@1": 0.9954565602836879, + "pass@2": 0.9967242292851691, + "pass@4": 0.9975758479784755, + "pass@8": 0.9980857085222107, + "pass@16": 0.9983981190211061, + "pass@32": 0.9986417677980687, + "pass@64": 0.9989868076836745, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.964855272861357, + "pass@2": 0.9744083600445962, + "pass@4": 0.980863755415529, + "pass@8": 0.9845045622809016, + "pass@16": 0.9864817601267433, + "pass@32": 0.988192742885706, + "pass@64": 0.9911504413060571, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985367063492063, + "pass@2": 0.99933172415948, + "pass@4": 0.999864562762988, + "pass@8": 0.999995076184736, + "pass@16": 0.999999996268659, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997688609467456, + "pass@2": 0.9999836200670922, + "pass@4": 0.9999999417602385, + "pass@8": 0.9999999999999069, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999824719551282, + "pass@2": 0.9999976340096912, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.997337147887324, + "pass@2": 0.9977261700121991, + "pass@4": 0.9981745219347582, + "pass@8": 0.9988346121771267, + "pass@16": 0.9995479293349335, + "pass@32": 0.9999462946812021, + "pass@64": 0.9999998252073432, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388437, + "answer_accuracy": 0.9943198107798165, + "avg_response_len": 198.04131256143512, + "resp_tokens_sum": 77366027, + "pass_at_k": { + "pass@1": 0.9943198107798165, + "pass@2": 0.9955572810781106, + "pass@4": 0.9964470693825461, + "pass@8": 0.9969549085780425, + "pass@16": 0.9971560486666131, + "pass@32": 0.9972767789698127, + "pass@64": 0.9974613265083974, + "pass@128": 0.9977064220183486 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569833205521472, + "pass@2": 0.9659142372349164, + "pass@4": 0.9719421844700393, + "pass@8": 0.9748675318830776, + "pass@16": 0.9757861748653367, + "pass@32": 0.9762266888126843, + "pass@64": 0.9769938650304216, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998414855072464, + "pass@2": 0.9999967904827115, + "pass@4": 0.9999999997282949, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999776146131805, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948283450704225, + "pass@2": 0.9955154153266054, + "pass@4": 0.9962789690901313, + "pass@8": 0.9971273245981712, + "pass@16": 0.997797475275284, + "pass@32": 0.9984192527437615, + "pass@64": 0.9993013197287653, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957662799401198, + "pass@2": 0.9974428603423076, + "pass@4": 0.9988750589372435, + "pass@8": 0.9997585789040461, + "pass@16": 0.9999876161726882, + "pass@32": 0.9999999824517899, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380437, + "answer_accuracy": 0.9950331645463676, + "avg_response_len": 211.4187782474054, + "resp_tokens_sum": 80833010, + "pass_at_k": { + "pass@1": 0.9950331645463676, + "pass@2": 0.9964548229334995, + "pass@4": 0.9975926760619602, + "pass@8": 0.9983688551648029, + "pass@16": 0.9988589881110809, + "pass@32": 0.9992246463711005, + "pass@64": 0.9995777785346197, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9654617537313432, + "pass@2": 0.974182483253026, + "pass@4": 0.9814832566638125, + "pass@8": 0.9865134381095033, + "pass@16": 0.990010567366735, + "pass@32": 0.9930920761226464, + "pass@64": 0.9962352974533647, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999080882352941, + "pass@2": 0.9999992762853173, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999258306962026, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954016516516516, + "pass@2": 0.9980015694828687, + "pass@4": 0.9994780266768457, + "pass@8": 0.9999541707109151, + "pass@16": 0.9999996450540588, + "pass@32": 0.9999999999945218, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.994599075562701, + "pass@2": 0.9959007962630074, + "pass@4": 0.9973834390074231, + "pass@8": 0.9989100635493507, + "pass@16": 0.9998018508583297, + "pass@32": 0.9999941260810753, + "pass@64": 0.9999999994727716, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.44683984375, + "avg_response_len": 327.42891796875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.44683984375, + "pass@2": 0.5119684670275582, + "pass@4": 0.569463077690281, + "pass@8": 0.6178828516789759, + "pass@16": 0.6568252672877612, + "pass@32": 0.6882615383682829, + "pass@64": 0.7149255891967156, + "pass@128": 0.7386 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.96253125, + "pass@2": 0.972941190944882, + "pass@4": 0.9790027408136479, + "pass@8": 0.9831148686678516, + "pass@16": 0.98574542765598, + "pass@32": 0.9875692041741516, + "pass@64": 0.9892254753754837, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.876703125, + "pass@2": 0.9067059547244101, + "pass@4": 0.9224254388826393, + "pass@8": 0.9343966760611316, + "pass@16": 0.9450926015777066, + "pass@32": 0.9542979874178255, + "pass@64": 0.9614691678418349, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.70778125, + "pass@2": 0.7821064222440945, + "pass@4": 0.8326372481252345, + "pass@8": 0.8670544873404723, + "pass@16": 0.890846691933782, + "pass@32": 0.9088359604915771, + "pass@64": 0.9236104944347704, + "pass@128": 0.936 + }, + "14": { + "pass@1": 0.5578828125, + "pass@2": 0.6460127952755912, + "pass@4": 0.714011713348332, + "pass@8": 0.7653732918196041, + "pass@16": 0.803843338086645, + "pass@32": 0.8336969542236903, + "pass@64": 0.856039598782845, + "pass@128": 0.872 + }, + "15": { + "pass@1": 0.38559375, + "pass@2": 0.4773902559055116, + "pass@4": 0.5553634880014996, + "pass@8": 0.6194323066082491, + "pass@16": 0.6699661651256025, + "pass@32": 0.7098046818744348, + "pass@64": 0.7443967566005082, + "pass@128": 0.777 + }, + "16": { + "pass@1": 0.23978125, + "pass@2": 0.3170354330708663, + "pass@4": 0.3916617366891629, + "pass@8": 0.4572465077324236, + "pass@16": 0.5126771573643457, + "pass@32": 0.5610609835923893, + "pass@64": 0.6070904239009969, + "pass@128": 0.651 + }, + "17": { + "pass@1": 0.2129765625, + "pass@2": 0.2896716289370081, + "pass@4": 0.3657562349081365, + "pass@8": 0.4328031981342337, + "pass@16": 0.48777685207242233, + "pass@32": 0.5337343412962802, + "pass@64": 0.5753223230388924, + "pass@128": 0.614 + }, + "18": { + "pass@1": 0.193296875, + "pass@2": 0.2601900836614175, + "pass@4": 0.3277885975815519, + "pass@8": 0.3916256997651364, + "pass@16": 0.4472835877005817, + "pass@32": 0.49205269327981904, + "pass@64": 0.5271340188878637, + "pass@128": 0.556 + }, + "19": { + "pass@1": 0.1764375, + "pass@2": 0.24655782480314958, + "pass@4": 0.3155220647731533, + "pass@8": 0.3751407394764381, + "pass@16": 0.4233042476747667, + "pass@32": 0.46136250070818885, + "pass@64": 0.4921356285231653, + "pass@128": 0.521 + }, + "20": { + "pass@1": 0.1554140625, + "pass@2": 0.22107308070866163, + "pass@4": 0.2904615137795272, + "pass@8": 0.3526407411842315, + "pass@16": 0.40171660368563283, + "pass@32": 0.44020007662447264, + "pass@64": 0.4728320045807315, + "pass@128": 0.502 + } + }, + "per_op_accuracy": { + "11": 0.96253125, + "12": 0.876703125, + "13": 0.70778125, + "14": 0.5578828125, + "15": 0.38559375, + "16": 0.23978125, + "17": 0.2129765625, + "18": 0.193296875, + "19": 0.1764375, + "20": 0.1554140625 + }, + "per_op_avg_response_len": { + "11": 299.1838515625, + "12": 307.2964296875, + "13": 318.4115390625, + "14": 333.137890625, + "15": 335.3022890625, + "16": 329.080421875, + "17": 338.9223984375, + "18": 336.208109375, + "19": 336.5952578125, + "20": 340.1509921875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.1838515625, + "12": 307.2964296875, + "13": 318.4115390625, + "14": 333.137890625, + "15": 335.3022890625, + "16": 329.080421875, + "17": 338.9223984375, + "18": 336.208109375, + "19": 336.5952578125, + "20": 340.1509921875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 199569, + "answer_accuracy": 0.46333813150074293, + "avg_response_len": 314.1041906575037, + "resp_tokens_sum": 135290957, + "pass_at_k": { + "pass@1": 0.46333813150074293, + "pass@2": 0.5296728934375414, + "pass@4": 0.5881637628413112, + "pass@8": 0.6371085247462108, + "pass@16": 0.6759604016198593, + "pass@32": 0.7069346470921534, + "pass@64": 0.7328271853390599, + "pass@128": 0.7551263001485884 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9568431303116147, + "pass@2": 0.966704122147621, + "pass@4": 0.9737702782725814, + "pass@8": 0.9797293556840062, + "pass@16": 0.9839171023985908, + "pass@32": 0.9864784015256126, + "pass@64": 0.9885864447521222, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8533501059322034, + "pass@2": 0.8813041477601309, + "pass@4": 0.8969065471688916, + "pass@8": 0.9103012428913164, + "pass@16": 0.9232330989994582, + "pass@32": 0.9356599220900689, + "pass@64": 0.9473396438691338, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.760384012539185, + "pass@2": 0.8294336077308518, + "pass@4": 0.874541486977137, + "pass@8": 0.9027689973847379, + "pass@16": 0.9198466852847369, + "pass@32": 0.9308948171804944, + "pass@64": 0.9399058726440269, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.6034852024922118, + "pass@2": 0.6929789259695341, + "pass@4": 0.7630072742075453, + "pass@8": 0.8164932601872855, + "pass@16": 0.8526563967709537, + "pass@32": 0.8763087593659933, + "pass@64": 0.8936422032813724, + "pass@128": 0.9065420560747663 + }, + "15": { + "pass@1": 0.4252619219653179, + "pass@2": 0.5253370209594465, + "pass@4": 0.6099591160209015, + "pass@8": 0.6784789856366449, + "pass@16": 0.7314473595873143, + "pass@32": 0.7713284427298922, + "pass@64": 0.8031345386516523, + "pass@128": 0.8294797687861272 + }, + "16": { + "pass@1": 0.26829637096774195, + "pass@2": 0.3590876619253239, + "pass@4": 0.44438377874136703, + "pass@8": 0.5160205150620084, + "pass@16": 0.572072708861716, + "pass@32": 0.6173416875891375, + "pass@64": 0.6591309169128403, + "pass@128": 0.6967741935483871 + }, + "17": { + "pass@1": 0.210305274566474, + "pass@2": 0.2880021534067633, + "pass@4": 0.36638274885075767, + "pass@8": 0.4354115772958514, + "pass@16": 0.4924820483803051, + "pass@32": 0.5414044485040482, + "pass@64": 0.5843553236837202, + "pass@128": 0.6242774566473989 + }, + "18": { + "pass@1": 0.19066964285714286, + "pass@2": 0.2540649606299211, + "pass@4": 0.3153792696448659, + "pass@8": 0.3744374956178368, + "pass@16": 0.4304686242139373, + "pass@32": 0.4799023237679476, + "pass@64": 0.5191885486106175, + "pass@128": 0.5485714285714286 + }, + "19": { + "pass@1": 0.17203922672672672, + "pass@2": 0.2452760339079237, + "pass@4": 0.31713847114831356, + "pass@8": 0.3807029920118585, + "pass@16": 0.43294372077217796, + "pass@32": 0.47306262676523364, + "pass@64": 0.5035309613007347, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.16786317567567569, + "pass@2": 0.23649646201319421, + "pass@4": 0.3076818460755471, + "pass@8": 0.3678730399252492, + "pass@16": 0.41342134802722214, + "pass@32": 0.450621229184501, + "pass@64": 0.48393762875113866, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 189989, + "answer_accuracy": 0.4413586269699673, + "avg_response_len": 322.06403787540887, + "resp_tokens_sum": 138636974, + "pass_at_k": { + "pass@1": 0.4413586269699673, + "pass@2": 0.5069858329762768, + "pass@4": 0.5649965586928776, + "pass@8": 0.6138581833337501, + "pass@16": 0.6533552733745646, + "pass@32": 0.6853306195144797, + "pass@64": 0.7123738268599717, + "pass@128": 0.736544751709783 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9643731174698795, + "pass@2": 0.9750883455080163, + "pass@4": 0.9803072865138847, + "pass@8": 0.9823661486047269, + "pass@16": 0.9832962407555205, + "pass@32": 0.9844320989052803, + "pass@64": 0.9860781709515227, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8863636363636364, + "pass@2": 0.9133642287660749, + "pass@4": 0.9258904760729354, + "pass@8": 0.9354300779654707, + "pass@16": 0.945221377305145, + "pass@32": 0.9540520042583904, + "pass@64": 0.9598380142135106, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.6867997198879552, + "pass@2": 0.7681042397273864, + "pass@4": 0.824483724986057, + "pass@8": 0.8627893286369971, + "pass@16": 0.8897924787212615, + "pass@32": 0.9118141519741789, + "pass@64": 0.9305813041850224, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.5445906432748538, + "pass@2": 0.6282847970483952, + "pass@4": 0.6919824232497251, + "pass@8": 0.7401227434606403, + "pass@16": 0.7772650766591268, + "pass@32": 0.8067142511689581, + "pass@64": 0.8289431092869494, + "pass@128": 0.8450292397660819 + }, + "15": { + "pass@1": 0.3677556818181818, + "pass@2": 0.4612525351944643, + "pass@4": 0.5395015228210107, + "pass@8": 0.6038984673310392, + "pass@16": 0.6548475782041733, + "pass@32": 0.695211253240757, + "pass@64": 0.7307349783554408, + "pass@128": 0.7636363636363637 + }, + "16": { + "pass@1": 0.23595530626780628, + "pass@2": 0.30808173217129914, + "pass@4": 0.37879288913672104, + "pass@8": 0.44199327843430775, + "pass@16": 0.49717687261406224, + "pass@32": 0.5475518095537767, + "pass@64": 0.5956511172414417, + "pass@128": 0.6410256410256411 + }, + "17": { + "pass@1": 0.23090547360248448, + "pass@2": 0.31294512948109743, + "pass@4": 0.3923752244478753, + "pass@8": 0.4620048777355182, + "pass@16": 0.5184937076070995, + "pass@32": 0.5641996443460395, + "pass@64": 0.6058905683329744, + "pass@128": 0.6490683229813664 + }, + "18": { + "pass@1": 0.19181985294117648, + "pass@2": 0.2639264416396479, + "pass@4": 0.3402037715873749, + "pass@8": 0.41292313703410055, + "pass@16": 0.4755867515992984, + "pass@32": 0.5242663505584598, + "pass@64": 0.5607374408176437, + "pass@128": 0.5911764705882353 + }, + "19": { + "pass@1": 0.17740885416666666, + "pass@2": 0.24643025578833894, + "pass@4": 0.3146818307867766, + "pass@8": 0.37225657621985075, + "pass@16": 0.4180985090398595, + "pass@32": 0.4553241566350863, + "pass@64": 0.48681976016382017, + "pass@128": 0.5178571428571429 + }, + "20": { + "pass@1": 0.1394320733532934, + "pass@2": 0.2010787725729644, + "pass@4": 0.26803790703183067, + "pass@8": 0.32950827537338695, + "pass@16": 0.37733200724855503, + "pass@32": 0.4121459313058152, + "pass@64": 0.43980782684006975, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 182397, + "answer_accuracy": 0.4355062843826406, + "avg_response_len": 346.6464604981663, + "resp_tokens_sum": 145181084, + "pass_at_k": { + "pass@1": 0.4355062843826406, + "pass@2": 0.49888203776254814, + "pass@4": 0.5548216039296309, + "pass@8": 0.6022473289936959, + "pass@16": 0.6407127558276247, + "pass@32": 0.6720701169897646, + "pass@64": 0.6991379075706123, + "pass@128": 0.7237163814180929 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9669642857142857, + "pass@2": 0.977667635295588, + "pass@4": 0.9834914713041819, + "pass@8": 0.987697919919454, + "pass@16": 0.9903756780268099, + "pass@32": 0.9920980050763724, + "pass@64": 0.9932587543558059, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8925602064220184, + "pass@2": 0.927709778106865, + "pass@4": 0.9466711294115757, + "pass@8": 0.9594735205095442, + "pass@16": 0.968631391961948, + "pass@32": 0.9747149407936225, + "pass@64": 0.9783565974863963, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6791087962962963, + "pass@2": 0.7509379252940609, + "pass@4": 0.8003636541960031, + "pass@8": 0.836590700158435, + "pass@16": 0.8634558161248805, + "pass@32": 0.8838359924574606, + "pass@64": 0.8998857884792347, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5279349035608308, + "pass@2": 0.6192675354564355, + "pass@4": 0.6896984853896232, + "pass@8": 0.7423055698396044, + "pass@16": 0.7843204109962838, + "pass@32": 0.8204913013870107, + "pass@64": 0.8477207957073809, + "pass@128": 0.8664688427299704 + }, + "15": { + "pass@1": 0.36140046296296297, + "pass@2": 0.4426244137017595, + "pass@4": 0.5132164547139938, + "pass@8": 0.5721977881442194, + "pass@16": 0.6197088824105395, + "pass@32": 0.658967059013711, + "pass@64": 0.6955853807708048, + "pass@128": 0.7345679012345679 + }, + "16": { + "pass@1": 0.21766685103244837, + "pass@2": 0.2878512385896452, + "pass@4": 0.35677422180634516, + "pass@8": 0.4192935319550409, + "pass@16": 0.4744116086421201, + "pass@32": 0.5235822279834264, + "pass@64": 0.5713460103429768, + "pass@128": 0.6194690265486725 + }, + "17": { + "pass@1": 0.19837161144578314, + "pass@2": 0.26883901254624787, + "pass@4": 0.33928608895650064, + "pass@8": 0.40176271614166426, + "pass@16": 0.4530815948293738, + "pass@32": 0.49619312239293656, + "pass@64": 0.536260897714176, + "pass@128": 0.5692771084337349 + }, + "18": { + "pass@1": 0.19788306451612903, + "pass@2": 0.2630076041402083, + "pass@4": 0.3281824866649735, + "pass@8": 0.38767325712032175, + "pass@16": 0.4352260441352987, + "pass@32": 0.47043974442309966, + "pass@64": 0.4992493451488665, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17987632175226587, + "pass@2": 0.247976856697671, + "pass@4": 0.3147488179347673, + "pass@8": 0.37247260884174876, + "pass@16": 0.41889090519709676, + "pass@32": 0.45572123675521775, + "pass@64": 0.4860676102567291, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.15899493243243243, + "pass@2": 0.2257040505071999, + "pass@4": 0.29573212638960666, + "pass@8": 0.3606103751183562, + "pass@16": 0.41446968279744983, + "pass@32": 0.457917316065741, + "pass@64": 0.49484972985591286, + "pass@128": 0.5285285285285285 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7064617598684211, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7064617598684211, + "pass@2": 0.7413598153232354, + "pass@4": 0.7720746653641805, + "pass@8": 0.7978418132148272, + "pass@16": 0.8184958716482886, + "pass@32": 0.835156126645237, + "pass@64": 0.8493287696055289, + "pass@128": 0.862 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9624921875, + "pass@2": 0.9715636072834645, + "pass@4": 0.9781628562054747, + "pass@8": 0.9820358637737919, + "pass@16": 0.984177149756922, + "pass@32": 0.9859330858922759, + "pass@64": 0.988238824249548, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999453125, + "pass@2": 0.9997881397637796, + "pass@4": 0.9999573371766028, + "pass@8": 0.9999984489981919, + "pass@16": 0.9999999988246276, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998984375, + "pass@2": 0.9999944635826771, + "pass@4": 0.9999999803149605, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.996578125, + "pass@2": 0.9977417568897637, + "pass@4": 0.998505216816648, + "pass@8": 0.9989649390790856, + "pass@16": 0.9992179855257273, + "pass@32": 0.9994388347222112, + "pass@64": 0.9997519685037116, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959609375, + "pass@2": 0.9970638533464568, + "pass@4": 0.9981624745031872, + "pass@8": 0.9991666824406795, + "pass@16": 0.9997737543325198, + "pass@32": 0.999979101961939, + "pass@64": 0.9999999377846387, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.96253125, + "pass@2": 0.972941190944882, + "pass@4": 0.9790027408136479, + "pass@8": 0.9831148686678516, + "pass@16": 0.98574542765598, + "pass@32": 0.9875692041741516, + "pass@64": 0.9892254753754837, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.876703125, + "pass@2": 0.9067059547244101, + "pass@4": 0.9224254388826393, + "pass@8": 0.9343966760611316, + "pass@16": 0.9450926015777066, + "pass@32": 0.9542979874178255, + "pass@64": 0.9614691678418349, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.70778125, + "pass@2": 0.7821064222440945, + "pass@4": 0.8326372481252345, + "pass@8": 0.8670544873404723, + "pass@16": 0.890846691933782, + "pass@32": 0.9088359604915771, + "pass@64": 0.9236104944347704, + "pass@128": 0.936 + }, + "14": { + "pass@1": 0.5578828125, + "pass@2": 0.6460127952755912, + "pass@4": 0.714011713348332, + "pass@8": 0.7653732918196041, + "pass@16": 0.803843338086645, + "pass@32": 0.8336969542236903, + "pass@64": 0.856039598782845, + "pass@128": 0.872 + }, + "15": { + "pass@1": 0.38559375, + "pass@2": 0.4773902559055116, + "pass@4": 0.5553634880014996, + "pass@8": 0.6194323066082491, + "pass@16": 0.6699661651256025, + "pass@32": 0.7098046818744348, + "pass@64": 0.7443967566005082, + "pass@128": 0.777 + }, + "16": { + "pass@1": 0.23978125, + "pass@2": 0.3170354330708663, + "pass@4": 0.3916617366891629, + "pass@8": 0.4572465077324236, + "pass@16": 0.5126771573643457, + "pass@32": 0.5610609835923893, + "pass@64": 0.6070904239009969, + "pass@128": 0.651 + }, + "17": { + "pass@1": 0.2129765625, + "pass@2": 0.2896716289370081, + "pass@4": 0.3657562349081365, + "pass@8": 0.4328031981342337, + "pass@16": 0.48777685207242233, + "pass@32": 0.5337343412962802, + "pass@64": 0.5753223230388924, + "pass@128": 0.614 + }, + "18": { + "pass@1": 0.193296875, + "pass@2": 0.2601900836614175, + "pass@4": 0.3277885975815519, + "pass@8": 0.3916256997651364, + "pass@16": 0.4472835877005817, + "pass@32": 0.49205269327981904, + "pass@64": 0.5271340188878637, + "pass@128": 0.556 + }, + "19": { + "pass@1": 0.1764375, + "pass@2": 0.24655782480314958, + "pass@4": 0.3155220647731533, + "pass@8": 0.3751407394764381, + "pass@16": 0.4233042476747667, + "pass@32": 0.46136250070818885, + "pass@64": 0.4921356285231653, + "pass@128": 0.521 + }, + "20": { + "pass@1": 0.1554140625, + "pass@2": 0.22107308070866163, + "pass@4": 0.2904615137795272, + "pass@8": 0.3526407411842315, + "pass@16": 0.40171660368563283, + "pass@32": 0.44020007662447264, + "pass@64": 0.4728320045807315, + "pass@128": 0.502 + } + }, + "per_op_accuracy": { + "10": 0.9624921875, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999453125, + "6": 0.9999921875, + "7": 0.9998984375, + "8": 0.996578125, + "9": 0.9959609375, + "11": 0.96253125, + "12": 0.876703125, + "13": 0.70778125, + "14": 0.5578828125, + "15": 0.38559375, + "16": 0.23978125, + "17": 0.2129765625, + "18": 0.193296875, + "19": 0.1764375, + "20": 0.1554140625 + }, + "per_op_avg_response_len": { + "10": 281.153859375, + "2": 115.629984375, + "3": 144.724578125, + "4": 159.8816328125, + "5": 172.73553125, + "6": 198.9245234375, + "7": 223.9551328125, + "8": 243.5811875, + "9": 259.5112734375, + "11": 299.1838515625, + "12": 307.2964296875, + "13": 318.4115390625, + "14": 333.137890625, + "15": 335.3022890625, + "16": 329.080421875, + "17": 338.9223984375, + "18": 336.208109375, + "19": 336.5952578125, + "20": 340.1509921875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.153859375, + "2": 115.629984375, + "3": 144.724578125, + "4": 159.8816328125, + "5": 172.73553125, + "6": 198.9245234375, + "7": 223.9551328125, + "8": 243.5811875, + "9": 259.5112734375, + "11": 299.1838515625, + "12": 307.2964296875, + "13": 318.4115390625, + "14": 333.137890625, + "15": 335.3022890625, + "16": 329.080421875, + "17": 338.9223984375, + "18": 336.208109375, + "19": 336.5952578125, + "20": 340.1509921875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 576855, + "answer_accuracy": 0.7124058943250079, + "avg_response_len": 256.26435790784063, + "resp_tokens_sum": 207504426, + "pass_at_k": { + "pass@1": 0.7124058943250079, + "pass@2": 0.7482848133624225, + "pass@4": 0.7797965772724061, + "pass@8": 0.8060704977403156, + "pass@16": 0.8268832725059063, + "pass@32": 0.8434735001446694, + "pass@64": 0.8574080645300883, + "pass@128": 0.8695858362314258 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.964855272861357, + "pass@2": 0.9744083600445962, + "pass@4": 0.980863755415529, + "pass@8": 0.9845045622809016, + "pass@16": 0.9864817601267433, + "pass@32": 0.988192742885706, + "pass@64": 0.9911504413060571, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985367063492063, + "pass@2": 0.99933172415948, + "pass@4": 0.999864562762988, + "pass@8": 0.999995076184736, + "pass@16": 0.999999996268659, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997688609467456, + "pass@2": 0.9999836200670922, + "pass@4": 0.9999999417602385, + "pass@8": 0.9999999999999069, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999824719551282, + "pass@2": 0.9999976340096912, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.997337147887324, + "pass@2": 0.9977261700121991, + "pass@4": 0.9981745219347582, + "pass@8": 0.9988346121771267, + "pass@16": 0.9995479293349335, + "pass@32": 0.9999462946812021, + "pass@64": 0.9999998252073432, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9568431303116147, + "pass@2": 0.966704122147621, + "pass@4": 0.9737702782725814, + "pass@8": 0.9797293556840062, + "pass@16": 0.9839171023985908, + "pass@32": 0.9864784015256126, + "pass@64": 0.9885864447521222, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8533501059322034, + "pass@2": 0.8813041477601309, + "pass@4": 0.8969065471688916, + "pass@8": 0.9103012428913164, + "pass@16": 0.9232330989994582, + "pass@32": 0.9356599220900689, + "pass@64": 0.9473396438691338, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.760384012539185, + "pass@2": 0.8294336077308518, + "pass@4": 0.874541486977137, + "pass@8": 0.9027689973847379, + "pass@16": 0.9198466852847369, + "pass@32": 0.9308948171804944, + "pass@64": 0.9399058726440269, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.6034852024922118, + "pass@2": 0.6929789259695341, + "pass@4": 0.7630072742075453, + "pass@8": 0.8164932601872855, + "pass@16": 0.8526563967709537, + "pass@32": 0.8763087593659933, + "pass@64": 0.8936422032813724, + "pass@128": 0.9065420560747663 + }, + "15": { + "pass@1": 0.4252619219653179, + "pass@2": 0.5253370209594465, + "pass@4": 0.6099591160209015, + "pass@8": 0.6784789856366449, + "pass@16": 0.7314473595873143, + "pass@32": 0.7713284427298922, + "pass@64": 0.8031345386516523, + "pass@128": 0.8294797687861272 + }, + "16": { + "pass@1": 0.26829637096774195, + "pass@2": 0.3590876619253239, + "pass@4": 0.44438377874136703, + "pass@8": 0.5160205150620084, + "pass@16": 0.572072708861716, + "pass@32": 0.6173416875891375, + "pass@64": 0.6591309169128403, + "pass@128": 0.6967741935483871 + }, + "17": { + "pass@1": 0.210305274566474, + "pass@2": 0.2880021534067633, + "pass@4": 0.36638274885075767, + "pass@8": 0.4354115772958514, + "pass@16": 0.4924820483803051, + "pass@32": 0.5414044485040482, + "pass@64": 0.5843553236837202, + "pass@128": 0.6242774566473989 + }, + "18": { + "pass@1": 0.19066964285714286, + "pass@2": 0.2540649606299211, + "pass@4": 0.3153792696448659, + "pass@8": 0.3744374956178368, + "pass@16": 0.4304686242139373, + "pass@32": 0.4799023237679476, + "pass@64": 0.5191885486106175, + "pass@128": 0.5485714285714286 + }, + "19": { + "pass@1": 0.17203922672672672, + "pass@2": 0.2452760339079237, + "pass@4": 0.31713847114831356, + "pass@8": 0.3807029920118585, + "pass@16": 0.43294372077217796, + "pass@32": 0.47306262676523364, + "pass@64": 0.5035309613007347, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.16786317567567569, + "pass@2": 0.23649646201319421, + "pass@4": 0.3076818460755471, + "pass@8": 0.3678730399252492, + "pass@16": 0.41342134802722214, + "pass@32": 0.450621229184501, + "pass@64": 0.48393762875113866, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 578426, + "answer_accuracy": 0.704435405300078, + "avg_response_len": 263.05899381332813, + "resp_tokens_sum": 216003001, + "pass_at_k": { + "pass@1": 0.704435405300078, + "pass@2": 0.7394285546608842, + "pass@4": 0.7702634267559836, + "pass@8": 0.7961202574484137, + "pass@16": 0.8169219087902136, + "pass@32": 0.8337421048859038, + "pass@64": 0.8480070379164071, + "pass@128": 0.8607950116913484 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569833205521472, + "pass@2": 0.9659142372349164, + "pass@4": 0.9719421844700393, + "pass@8": 0.9748675318830776, + "pass@16": 0.9757861748653367, + "pass@32": 0.9762266888126843, + "pass@64": 0.9769938650304216, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998414855072464, + "pass@2": 0.9999967904827115, + "pass@4": 0.9999999997282949, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999776146131805, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948283450704225, + "pass@2": 0.9955154153266054, + "pass@4": 0.9962789690901313, + "pass@8": 0.9971273245981712, + "pass@16": 0.997797475275284, + "pass@32": 0.9984192527437615, + "pass@64": 0.9993013197287653, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957662799401198, + "pass@2": 0.9974428603423076, + "pass@4": 0.9988750589372435, + "pass@8": 0.9997585789040461, + "pass@16": 0.9999876161726882, + "pass@32": 0.9999999824517899, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9643731174698795, + "pass@2": 0.9750883455080163, + "pass@4": 0.9803072865138847, + "pass@8": 0.9823661486047269, + "pass@16": 0.9832962407555205, + "pass@32": 0.9844320989052803, + "pass@64": 0.9860781709515227, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8863636363636364, + "pass@2": 0.9133642287660749, + "pass@4": 0.9258904760729354, + "pass@8": 0.9354300779654707, + "pass@16": 0.945221377305145, + "pass@32": 0.9540520042583904, + "pass@64": 0.9598380142135106, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.6867997198879552, + "pass@2": 0.7681042397273864, + "pass@4": 0.824483724986057, + "pass@8": 0.8627893286369971, + "pass@16": 0.8897924787212615, + "pass@32": 0.9118141519741789, + "pass@64": 0.9305813041850224, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.5445906432748538, + "pass@2": 0.6282847970483952, + "pass@4": 0.6919824232497251, + "pass@8": 0.7401227434606403, + "pass@16": 0.7772650766591268, + "pass@32": 0.8067142511689581, + "pass@64": 0.8289431092869494, + "pass@128": 0.8450292397660819 + }, + "15": { + "pass@1": 0.3677556818181818, + "pass@2": 0.4612525351944643, + "pass@4": 0.5395015228210107, + "pass@8": 0.6038984673310392, + "pass@16": 0.6548475782041733, + "pass@32": 0.695211253240757, + "pass@64": 0.7307349783554408, + "pass@128": 0.7636363636363637 + }, + "16": { + "pass@1": 0.23595530626780628, + "pass@2": 0.30808173217129914, + "pass@4": 0.37879288913672104, + "pass@8": 0.44199327843430775, + "pass@16": 0.49717687261406224, + "pass@32": 0.5475518095537767, + "pass@64": 0.5956511172414417, + "pass@128": 0.6410256410256411 + }, + "17": { + "pass@1": 0.23090547360248448, + "pass@2": 0.31294512948109743, + "pass@4": 0.3923752244478753, + "pass@8": 0.4620048777355182, + "pass@16": 0.5184937076070995, + "pass@32": 0.5641996443460395, + "pass@64": 0.6058905683329744, + "pass@128": 0.6490683229813664 + }, + "18": { + "pass@1": 0.19181985294117648, + "pass@2": 0.2639264416396479, + "pass@4": 0.3402037715873749, + "pass@8": 0.41292313703410055, + "pass@16": 0.4755867515992984, + "pass@32": 0.5242663505584598, + "pass@64": 0.5607374408176437, + "pass@128": 0.5911764705882353 + }, + "19": { + "pass@1": 0.17740885416666666, + "pass@2": 0.24643025578833894, + "pass@4": 0.3146818307867766, + "pass@8": 0.37225657621985075, + "pass@16": 0.4180985090398595, + "pass@32": 0.4553241566350863, + "pass@64": 0.48681976016382017, + "pass@128": 0.5178571428571429 + }, + "20": { + "pass@1": 0.1394320733532934, + "pass@2": 0.2010787725729644, + "pass@4": 0.26803790703183067, + "pass@8": 0.32950827537338695, + "pass@16": 0.37733200724855503, + "pass@32": 0.4121459313058152, + "pass@64": 0.43980782684006975, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 562834, + "answer_accuracy": 0.7025308555679821, + "avg_response_len": 282.1113771169516, + "resp_tokens_sum": 226014094, + "pass_at_k": { + "pass@1": 0.7025308555679821, + "pass@2": 0.736340083665345, + "pass@4": 0.7661264757077474, + "pass@8": 0.791289508043559, + "pass@16": 0.8116318796222768, + "pass@32": 0.8281989841030529, + "pass@64": 0.8425176638526883, + "pass@128": 0.8555679821057677 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9654617537313432, + "pass@2": 0.974182483253026, + "pass@4": 0.9814832566638125, + "pass@8": 0.9865134381095033, + "pass@16": 0.990010567366735, + "pass@32": 0.9930920761226464, + "pass@64": 0.9962352974533647, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999080882352941, + "pass@2": 0.9999992762853173, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999258306962026, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954016516516516, + "pass@2": 0.9980015694828687, + "pass@4": 0.9994780266768457, + "pass@8": 0.9999541707109151, + "pass@16": 0.9999996450540588, + "pass@32": 0.9999999999945218, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.994599075562701, + "pass@2": 0.9959007962630074, + "pass@4": 0.9973834390074231, + "pass@8": 0.9989100635493507, + "pass@16": 0.9998018508583297, + "pass@32": 0.9999941260810753, + "pass@64": 0.9999999994727716, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9669642857142857, + "pass@2": 0.977667635295588, + "pass@4": 0.9834914713041819, + "pass@8": 0.987697919919454, + "pass@16": 0.9903756780268099, + "pass@32": 0.9920980050763724, + "pass@64": 0.9932587543558059, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8925602064220184, + "pass@2": 0.927709778106865, + "pass@4": 0.9466711294115757, + "pass@8": 0.9594735205095442, + "pass@16": 0.968631391961948, + "pass@32": 0.9747149407936225, + "pass@64": 0.9783565974863963, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6791087962962963, + "pass@2": 0.7509379252940609, + "pass@4": 0.8003636541960031, + "pass@8": 0.836590700158435, + "pass@16": 0.8634558161248805, + "pass@32": 0.8838359924574606, + "pass@64": 0.8998857884792347, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5279349035608308, + "pass@2": 0.6192675354564355, + "pass@4": 0.6896984853896232, + "pass@8": 0.7423055698396044, + "pass@16": 0.7843204109962838, + "pass@32": 0.8204913013870107, + "pass@64": 0.8477207957073809, + "pass@128": 0.8664688427299704 + }, + "15": { + "pass@1": 0.36140046296296297, + "pass@2": 0.4426244137017595, + "pass@4": 0.5132164547139938, + "pass@8": 0.5721977881442194, + "pass@16": 0.6197088824105395, + "pass@32": 0.658967059013711, + "pass@64": 0.6955853807708048, + "pass@128": 0.7345679012345679 + }, + "16": { + "pass@1": 0.21766685103244837, + "pass@2": 0.2878512385896452, + "pass@4": 0.35677422180634516, + "pass@8": 0.4192935319550409, + "pass@16": 0.4744116086421201, + "pass@32": 0.5235822279834264, + "pass@64": 0.5713460103429768, + "pass@128": 0.6194690265486725 + }, + "17": { + "pass@1": 0.19837161144578314, + "pass@2": 0.26883901254624787, + "pass@4": 0.33928608895650064, + "pass@8": 0.40176271614166426, + "pass@16": 0.4530815948293738, + "pass@32": 0.49619312239293656, + "pass@64": 0.536260897714176, + "pass@128": 0.5692771084337349 + }, + "18": { + "pass@1": 0.19788306451612903, + "pass@2": 0.2630076041402083, + "pass@4": 0.3281824866649735, + "pass@8": 0.38767325712032175, + "pass@16": 0.4352260441352987, + "pass@32": 0.47043974442309966, + "pass@64": 0.4992493451488665, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17987632175226587, + "pass@2": 0.247976856697671, + "pass@4": 0.3147488179347673, + "pass@8": 0.37247260884174876, + "pass@16": 0.41889090519709676, + "pass@32": 0.45572123675521775, + "pass@64": 0.4860676102567291, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.15899493243243243, + "pass@2": 0.2257040505071999, + "pass@4": 0.29573212638960666, + "pass@8": 0.3606103751183562, + "pass@16": 0.41446968279744983, + "pass@32": 0.457917316065741, + "pass@64": 0.49484972985591286, + "pass@128": 0.5285285285285285 + } + } + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl b/Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a225c2ab0763b9b69073bae2943638061a972a07 --- /dev/null +++ b/Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abdb12f6d038b927e060a53ec87dfd2305d556b14c3e117c4aad58035dc74fc +size 2568335013 diff --git a/Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl b/Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7753ced51c8920c86c66f006a75f524a7a2cbeca --- /dev/null +++ b/Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0faae406ec3d46534b09ef38089b687062513477f9ef33d60d9610f09b4b1fa +size 443430559 diff --git a/Qwen3-100M-ep4/composition/graphs/composition_score_report.jsonl b/Qwen3-100M-ep4/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..03b2e5ad89634785097ff01420f3fc5a31a8bc48 --- /dev/null +++ b/Qwen3-100M-ep4/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-ep4/composition", "results_dir": "results/Qwen3-100M-ep4/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1026138, "value_mismatch": 6602, "dependency_mismatch": 124380, "answer_mismatch": 5840, "extra_nodes": 8258, "missing_nodes": 8221, "perfect_with_extra": 86, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 338323, "value_mismatch": 2010, "answer_mismatch": 1722, "dependency_mismatch": 40374, "extra_nodes": 2856, "missing_nodes": 2819, "perfect_with_extra": 16}, "teachers_in_school": {"total": 390656, "perfect": 347450, "value_mismatch": 2410, "dependency_mismatch": 42575, "answer_mismatch": 2219, "missing_nodes": 2895, "extra_nodes": 2905, "perfect_with_extra": 8}, "movie_festival_awards": {"total": 382336, "perfect": 340365, "dependency_mismatch": 41431, "value_mismatch": 2182, "answer_mismatch": 1899, "extra_nodes": 2497, "missing_nodes": 2507, "perfect_with_extra": 62}}, "results_path": "results/Qwen3-100M-ep4/composition/graphs/checkpoint-85276_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 258020, "value_mismatch": 824338, "dependency_mismatch": 998879, "answer_mismatch": 708052, "extra_nodes": 67333, "missing_nodes": 765676, "perfect_with_extra": 19, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 270906, "dependency_mismatch": 331115, "answer_mismatch": 231155, "perfect": 91646, "extra_nodes": 25775, "missing_nodes": 243669, "perfect_with_extra": 4}, "movie_festival_awards": {"total": 418816, "perfect": 80786, "value_mismatch": 274419, "dependency_mismatch": 330693, "answer_mismatch": 236421, "missing_nodes": 258294, "extra_nodes": 21013, "perfect_with_extra": 15}, "teachers_in_school": {"total": 430464, "perfect": 85588, "value_mismatch": 279013, "answer_mismatch": 240476, "dependency_mismatch": 337071, "extra_nodes": 20545, "missing_nodes": 263713}}, "results_path": "results/Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1284158, "value_mismatch": 830940, "dependency_mismatch": 1123259, "answer_mismatch": 713892, "extra_nodes": 75591, "missing_nodes": 773897, "perfect_with_extra": 105, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-ep4/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 429969, "value_mismatch": 272916, "answer_mismatch": 232877, "dependency_mismatch": 371489, "extra_nodes": 28631, "missing_nodes": 246488, "perfect_with_extra": 20}, "teachers_in_school": {"total": 821120, "perfect": 433038, "value_mismatch": 281423, "dependency_mismatch": 379646, "answer_mismatch": 242695, "missing_nodes": 266608, "extra_nodes": 23450, "perfect_with_extra": 8}, "movie_festival_awards": {"total": 801152, "perfect": 421151, "dependency_mismatch": 372124, "value_mismatch": 276601, "answer_mismatch": 238320, "extra_nodes": 23510, "missing_nodes": 260801, "perfect_with_extra": 77}}}} diff --git a/Qwen3-100M-ep4/composition/graphs/results.json b/Qwen3-100M-ep4/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..5fe489934d714b7c7a6dfe0961d795d2ad683b24 --- /dev/null +++ b/Qwen3-100M-ep4/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-85276_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.996875, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9555555555555556, + "pass@2": 0.9746031746031746, + "pass@4": 0.9904761904761905, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9640718562874252, + "pass@2": 0.9850299401197605, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8994082840236687, + "pass@2": 0.9437869822485208, + "pass@4": 0.9644970414201184, + "pass@8": 0.985207100591716, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8301282051282052, + "pass@2": 0.8910256410256411, + "pass@4": 0.9391025641025641, + "pass@8": 0.9743589743589743, + "pass@16": 0.9903846153846154, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7436619718309859, + "pass@2": 0.819718309859155, + "pass@4": 0.9014084507042254, + "pass@8": 0.9577464788732394, + "pass@16": 0.9915492957746479, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6666666666666666, + "pass@2": 0.7374631268436578, + "pass@4": 0.8082595870206489, + "pass@8": 0.8790560471976401, + "pass@16": 0.9262536873156342, + "pass@32": 0.9557522123893806, + "pass@64": 0.9646017699115044, + "pass@128": 0.9734513274336283 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9555555555555556, + "6": 0.9640718562874252, + "7": 0.8994082840236687, + "8": 0.8301282051282052, + "9": 0.7436619718309859, + "10": 0.6666666666666666 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9746031746031746, + "6": 0.9850299401197605, + "7": 0.9437869822485208, + "8": 0.8910256410256411, + "9": 0.819718309859155, + "10": 0.7374631268436578 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9904761904761905, + "6": 1.0, + "7": 0.9644970414201184, + "8": 0.9391025641025641, + "9": 0.9014084507042254, + "10": 0.8082595870206489 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.985207100591716, + "8": 0.9743589743589743, + "9": 0.9577464788732394, + "10": 0.8790560471976401 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9903846153846154, + "9": 0.9915492957746479, + "10": 0.9262536873156342 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9557522123893806 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9646017699115044 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9734513274336283 + } + }, + "total_pass_at_k": { + "pass@1": 0.8922661263086795, + "pass@2": 0.9257007767646066, + "pass@4": 0.9547450185748058, + "pass@8": 0.9763593380614657, + "pass@16": 0.9888551165146909, + "pass@32": 0.994258696386356, + "pass@64": 0.9956095913542722, + "pass@128": 0.9966227625802094 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9558823529411765, + "pass@2": 0.9705882352941176, + "pass@4": 0.9941176470588236, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684542586750788, + "pass@2": 0.9747634069400631, + "pass@4": 0.9873817034700315, + "pass@8": 0.9936908517350158, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.879746835443038, + "pass@2": 0.9145569620253164, + "pass@4": 0.9493670886075949, + "pass@8": 0.9841772151898734, + "pass@16": 0.9968354430379747, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8048048048048048, + "pass@2": 0.8678678678678678, + "pass@4": 0.93993993993994, + "pass@8": 0.96996996996997, + "pass@16": 0.996996996996997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7620578778135049, + "pass@2": 0.8392282958199357, + "pass@4": 0.9003215434083601, + "pass@8": 0.9453376205787781, + "pass@16": 0.9678456591639871, + "pass@32": 0.9935691318327974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.608955223880597, + "pass@2": 0.6746268656716418, + "pass@4": 0.7671641791044777, + "pass@8": 0.8567164179104477, + "pass@16": 0.9074626865671642, + "pass@32": 0.9432835820895522, + "pass@64": 0.9641791044776119, + "pass@128": 0.9671641791044776 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9558823529411765, + "6": 0.9684542586750788, + "7": 0.879746835443038, + "8": 0.8048048048048048, + "9": 0.7620578778135049, + "10": 0.608955223880597 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9705882352941176, + "6": 0.9747634069400631, + "7": 0.9145569620253164, + "8": 0.8678678678678678, + "9": 0.8392282958199357, + "10": 0.6746268656716418 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 0.9873817034700315, + "7": 0.9493670886075949, + "8": 0.93993993993994, + "9": 0.9003215434083601, + "10": 0.7671641791044777 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9936908517350158, + "7": 0.9841772151898734, + "8": 0.96996996996997, + "9": 0.9453376205787781, + "10": 0.8567164179104477 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9678456591639871, + "10": 0.9074626865671642 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9935691318327974, + "10": 0.9432835820895522 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9641791044776119 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9671641791044776 + } + }, + "total_pass_at_k": { + "pass@1": 0.8875125544024104, + "pass@2": 0.916973552058922, + "pass@4": 0.9494476062939404, + "pass@8": 0.9725477067291597, + "pass@16": 0.9856042852360227, + "pass@32": 0.9929695346501507, + "pass@64": 0.9959825912286575, + "pass@128": 0.9963173752929361 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 0.985207100591716, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9797101449275363, + "pass@4": 0.991304347826087, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684813753581661, + "pass@2": 0.9770773638968482, + "pass@4": 0.9856733524355301, + "pass@8": 0.997134670487106, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8728323699421965, + "pass@2": 0.9132947976878613, + "pass@4": 0.9566473988439307, + "pass@8": 0.9913294797687862, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8112676056338028, + "pass@2": 0.8816901408450705, + "pass@4": 0.9605633802816902, + "pass@8": 0.9859154929577465, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7305389221556886, + "pass@2": 0.7724550898203593, + "pass@4": 0.8622754491017964, + "pass@8": 0.9401197604790419, + "pass@16": 0.9670658682634731, + "pass@32": 0.9910179640718563, + "pass@64": 0.9970059880239521, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6717791411042945, + "pass@2": 0.7392638036809815, + "pass@4": 0.8159509202453987, + "pass@8": 0.8926380368098159, + "pass@16": 0.9325153374233128, + "pass@32": 0.9478527607361963, + "pass@64": 0.9631901840490797, + "pass@128": 0.9662576687116564 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9594202898550724, + "6": 0.9684813753581661, + "7": 0.8728323699421965, + "8": 0.8112676056338028, + "9": 0.7305389221556886, + "10": 0.6717791411042945 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.985207100591716, + "5": 0.9797101449275363, + "6": 0.9770773638968482, + "7": 0.9132947976878613, + "8": 0.8816901408450705, + "9": 0.7724550898203593, + "10": 0.7392638036809815 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9856733524355301, + "7": 0.9566473988439307, + "8": 0.9605633802816902, + "9": 0.8622754491017964, + "10": 0.8159509202453987 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9913294797687862, + "8": 0.9859154929577465, + "9": 0.9401197604790419, + "10": 0.8926380368098159 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9670658682634731, + "10": 0.9325153374233128 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.9478527607361963 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9970059880239521, + "10": 0.9631901840490797 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 1.0, + "10": 0.9662576687116564 + } + }, + "total_pass_at_k": { + "pass@1": 0.8889252948885976, + "pass@2": 0.9171035386631717, + "pass@4": 0.9528178243774574, + "pass@8": 0.9790301441677588, + "pass@16": 0.9885321100917431, + "pass@32": 0.9931192660550459, + "pass@64": 0.9954128440366973, + "pass@128": 0.9960681520314548 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.989, + "pass@2": 0.994, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.957, + "pass@2": 0.975, + "pass@4": 0.992, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.967, + "pass@2": 0.979, + "pass@4": 0.991, + "pass@8": 0.997, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.884, + "pass@2": 0.924, + "pass@4": 0.957, + "pass@8": 0.987, + "pass@16": 0.997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.815, + "pass@2": 0.88, + "pass@4": 0.947, + "pass@8": 0.977, + "pass@16": 0.995, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.745, + "pass@2": 0.81, + "pass@4": 0.888, + "pass@8": 0.948, + "pass@16": 0.976, + "pass@32": 0.994, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.649, + "pass@2": 0.717, + "pass@4": 0.797, + "pass@8": 0.876, + "pass@16": 0.922, + "pass@32": 0.949, + "pass@64": 0.964, + "pass@128": 0.969 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.989, + "5": 0.957, + "6": 0.967, + "7": 0.884, + "8": 0.815, + "9": 0.745, + "10": 0.649 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.975, + "6": 0.979, + "7": 0.924, + "8": 0.88, + "9": 0.81, + "10": 0.717 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.992, + "6": 0.991, + "7": 0.957, + "8": 0.947, + "9": 0.888, + "10": 0.797 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.997, + "7": 0.987, + "8": 0.977, + "9": 0.948, + "10": 0.876 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.997, + "8": 0.995, + "9": 0.976, + "10": 0.922 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.994, + "10": 0.949 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.964 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.969 + } + }, + "total_pass_at_k": { + "pass@1": 0.8895555555555555, + "pass@2": 0.9198888888888889, + "pass@4": 0.9523333333333334, + "pass@8": 0.976, + "pass@16": 0.9876666666666667, + "pass@32": 0.9934444444444445, + "pass@64": 0.9956666666666667, + "pass@128": 0.9963333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-85276_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5835694050991501, + "pass@2": 0.6458923512747875, + "pass@4": 0.71671388101983, + "pass@8": 0.8158640226628895, + "pass@16": 0.8923512747875354, + "pass@32": 0.9206798866855525, + "pass@64": 0.943342776203966, + "pass@128": 0.9461756373937678 + }, + "12": { + "pass@1": 0.5790960451977402, + "pass@2": 0.6497175141242938, + "pass@4": 0.7033898305084746, + "pass@8": 0.7598870056497176, + "pass@16": 0.8107344632768362, + "pass@32": 0.8361581920903954, + "pass@64": 0.8502824858757062, + "pass@128": 0.867231638418079 + }, + "13": { + "pass@1": 0.4231974921630094, + "pass@2": 0.48589341692789967, + "pass@4": 0.5642633228840125, + "pass@8": 0.6238244514106583, + "pass@16": 0.6865203761755486, + "pass@32": 0.7366771159874608, + "pass@64": 0.774294670846395, + "pass@128": 0.7962382445141066 + }, + "14": { + "pass@1": 0.26479750778816197, + "pass@2": 0.3426791277258567, + "pass@4": 0.40186915887850466, + "pass@8": 0.4797507788161994, + "pass@16": 0.5545171339563862, + "pass@32": 0.6074766355140186, + "pass@64": 0.6697819314641744, + "pass@128": 0.7040498442367601 + }, + "15": { + "pass@1": 0.15895953757225434, + "pass@2": 0.1936416184971098, + "pass@4": 0.23699421965317918, + "pass@8": 0.2658959537572254, + "pass@16": 0.33236994219653176, + "pass@32": 0.3901734104046243, + "pass@64": 0.45375722543352603, + "pass@128": 0.49710982658959535 + }, + "16": { + "pass@1": 0.04838709677419355, + "pass@2": 0.08387096774193549, + "pass@4": 0.1064516129032258, + "pass@8": 0.12903225806451613, + "pass@16": 0.14838709677419354, + "pass@32": 0.16451612903225807, + "pass@64": 0.19032258064516128, + "pass@128": 0.2129032258064516 + }, + "17": { + "pass@1": 0.031791907514450865, + "pass@2": 0.04046242774566474, + "pass@4": 0.057803468208092484, + "pass@8": 0.06069364161849711, + "pass@16": 0.08092485549132948, + "pass@32": 0.11560693641618497, + "pass@64": 0.1329479768786127, + "pass@128": 0.15895953757225434 + }, + "18": { + "pass@1": 0.008571428571428572, + "pass@2": 0.011428571428571429, + "pass@4": 0.014285714285714285, + "pass@8": 0.02, + "pass@16": 0.02857142857142857, + "pass@32": 0.037142857142857144, + "pass@64": 0.04285714285714286, + "pass@128": 0.05142857142857143 + }, + "19": { + "pass@1": 0.003003003003003003, + "pass@2": 0.009009009009009009, + "pass@4": 0.009009009009009009, + "pass@8": 0.012012012012012012, + "pass@16": 0.012012012012012012, + "pass@32": 0.015015015015015015, + "pass@64": 0.018018018018018018, + "pass@128": 0.018018018018018018 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.003003003003003003, + "pass@16": 0.003003003003003003, + "pass@32": 0.003003003003003003, + "pass@64": 0.003003003003003003, + "pass@128": 0.003003003003003003 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5835694050991501, + "12": 0.5790960451977402, + "13": 0.4231974921630094, + "14": 0.26479750778816197, + "15": 0.15895953757225434, + "16": 0.04838709677419355, + "17": 0.031791907514450865, + "18": 0.008571428571428572, + "19": 0.003003003003003003, + "20": 0.0 + }, + "pass@2": { + "11": 0.6458923512747875, + "12": 0.6497175141242938, + "13": 0.48589341692789967, + "14": 0.3426791277258567, + "15": 0.1936416184971098, + "16": 0.08387096774193549, + "17": 0.04046242774566474, + "18": 0.011428571428571429, + "19": 0.009009009009009009, + "20": 0.0 + }, + "pass@4": { + "11": 0.71671388101983, + "12": 0.7033898305084746, + "13": 0.5642633228840125, + "14": 0.40186915887850466, + "15": 0.23699421965317918, + "16": 0.1064516129032258, + "17": 0.057803468208092484, + "18": 0.014285714285714285, + "19": 0.009009009009009009, + "20": 0.0 + }, + "pass@8": { + "11": 0.8158640226628895, + "12": 0.7598870056497176, + "13": 0.6238244514106583, + "14": 0.4797507788161994, + "15": 0.2658959537572254, + "16": 0.12903225806451613, + "17": 0.06069364161849711, + "18": 0.02, + "19": 0.012012012012012012, + "20": 0.003003003003003003 + }, + "pass@16": { + "11": 0.8923512747875354, + "12": 0.8107344632768362, + "13": 0.6865203761755486, + "14": 0.5545171339563862, + "15": 0.33236994219653176, + "16": 0.14838709677419354, + "17": 0.08092485549132948, + "18": 0.02857142857142857, + "19": 0.012012012012012012, + "20": 0.003003003003003003 + }, + "pass@32": { + "11": 0.9206798866855525, + "12": 0.8361581920903954, + "13": 0.7366771159874608, + "14": 0.6074766355140186, + "15": 0.3901734104046243, + "16": 0.16451612903225807, + "17": 0.11560693641618497, + "18": 0.037142857142857144, + "19": 0.015015015015015015, + "20": 0.003003003003003003 + }, + "pass@64": { + "11": 0.943342776203966, + "12": 0.8502824858757062, + "13": 0.774294670846395, + "14": 0.6697819314641744, + "15": 0.45375722543352603, + "16": 0.19032258064516128, + "17": 0.1329479768786127, + "18": 0.04285714285714286, + "19": 0.018018018018018018, + "20": 0.003003003003003003 + }, + "pass@128": { + "11": 0.9461756373937678, + "12": 0.867231638418079, + "13": 0.7962382445141066, + "14": 0.7040498442367601, + "15": 0.49710982658959535, + "16": 0.2129032258064516, + "17": 0.15895953757225434, + "18": 0.05142857142857143, + "19": 0.018018018018018018, + "20": 0.003003003003003003 + } + }, + "total_pass_at_k": { + "pass@1": 0.21277860326894502, + "pass@2": 0.2487369985141159, + "pass@4": 0.2835066864784547, + "pass@8": 0.31946508172362553, + "pass@16": 0.3575037147102526, + "pass@32": 0.38514115898959883, + "pass@64": 0.4101040118870728, + "pass@128": 0.4276374442793462 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5873015873015873, + "pass@2": 0.6634920634920635, + "pass@4": 0.7555555555555555, + "pass@8": 0.8539682539682539, + "pass@16": 0.926984126984127, + "pass@32": 0.9619047619047619, + "pass@64": 0.9714285714285714, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.5902140672782875, + "pass@2": 0.6238532110091743, + "pass@4": 0.6880733944954128, + "pass@8": 0.7706422018348624, + "pass@16": 0.8348623853211009, + "pass@32": 0.8746177370030581, + "pass@64": 0.8868501529051988, + "pass@128": 0.8929663608562691 + }, + "13": { + "pass@1": 0.41358024691358025, + "pass@2": 0.4537037037037037, + "pass@4": 0.5339506172839507, + "pass@8": 0.5925925925925926, + "pass@16": 0.6419753086419753, + "pass@32": 0.6666666666666666, + "pass@64": 0.7006172839506173, + "pass@128": 0.7160493827160493 + }, + "14": { + "pass@1": 0.21364985163204747, + "pass@2": 0.2789317507418398, + "pass@4": 0.3353115727002967, + "pass@8": 0.4035608308605341, + "pass@16": 0.4658753709198813, + "pass@32": 0.516320474777448, + "pass@64": 0.5548961424332344, + "pass@128": 0.6320474777448071 + }, + "15": { + "pass@1": 0.08024691358024691, + "pass@2": 0.11728395061728394, + "pass@4": 0.15123456790123457, + "pass@8": 0.19135802469135801, + "pass@16": 0.24074074074074073, + "pass@32": 0.2808641975308642, + "pass@64": 0.3549382716049383, + "pass@128": 0.404320987654321 + }, + "16": { + "pass@1": 0.038348082595870206, + "pass@2": 0.061946902654867256, + "pass@4": 0.06489675516224189, + "pass@8": 0.07669616519174041, + "pass@16": 0.09144542772861357, + "pass@32": 0.11504424778761062, + "pass@64": 0.13274336283185842, + "pass@128": 0.14749262536873156 + }, + "17": { + "pass@1": 0.02108433734939759, + "pass@2": 0.030120481927710843, + "pass@4": 0.03614457831325301, + "pass@8": 0.045180722891566265, + "pass@16": 0.05120481927710843, + "pass@32": 0.060240963855421686, + "pass@64": 0.06927710843373494, + "pass@128": 0.08433734939759036 + }, + "18": { + "pass@1": 0.012903225806451613, + "pass@2": 0.012903225806451613, + "pass@4": 0.016129032258064516, + "pass@8": 0.025806451612903226, + "pass@16": 0.03225806451612903, + "pass@32": 0.041935483870967745, + "pass@64": 0.04838709677419355, + "pass@128": 0.06451612903225806 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0030211480362537764, + "pass@8": 0.006042296072507553, + "pass@16": 0.00906344410876133, + "pass@32": 0.00906344410876133, + "pass@64": 0.00906344410876133, + "pass@128": 0.012084592145015106 + }, + "20": { + "pass@1": 0.003003003003003003, + "pass@2": 0.003003003003003003, + "pass@4": 0.006006006006006006, + "pass@8": 0.009009009009009009, + "pass@16": 0.012012012012012012, + "pass@32": 0.015015015015015015, + "pass@64": 0.015015015015015015, + "pass@128": 0.018018018018018018 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5873015873015873, + "12": 0.5902140672782875, + "13": 0.41358024691358025, + "14": 0.21364985163204747, + "15": 0.08024691358024691, + "16": 0.038348082595870206, + "17": 0.02108433734939759, + "18": 0.012903225806451613, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@2": { + "11": 0.6634920634920635, + "12": 0.6238532110091743, + "13": 0.4537037037037037, + "14": 0.2789317507418398, + "15": 0.11728395061728394, + "16": 0.061946902654867256, + "17": 0.030120481927710843, + "18": 0.012903225806451613, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@4": { + "11": 0.7555555555555555, + "12": 0.6880733944954128, + "13": 0.5339506172839507, + "14": 0.3353115727002967, + "15": 0.15123456790123457, + "16": 0.06489675516224189, + "17": 0.03614457831325301, + "18": 0.016129032258064516, + "19": 0.0030211480362537764, + "20": 0.006006006006006006 + }, + "pass@8": { + "11": 0.8539682539682539, + "12": 0.7706422018348624, + "13": 0.5925925925925926, + "14": 0.4035608308605341, + "15": 0.19135802469135801, + "16": 0.07669616519174041, + "17": 0.045180722891566265, + "18": 0.025806451612903226, + "19": 0.006042296072507553, + "20": 0.009009009009009009 + }, + "pass@16": { + "11": 0.926984126984127, + "12": 0.8348623853211009, + "13": 0.6419753086419753, + "14": 0.4658753709198813, + "15": 0.24074074074074073, + "16": 0.09144542772861357, + "17": 0.05120481927710843, + "18": 0.03225806451612903, + "19": 0.00906344410876133, + "20": 0.012012012012012012 + }, + "pass@32": { + "11": 0.9619047619047619, + "12": 0.8746177370030581, + "13": 0.6666666666666666, + "14": 0.516320474777448, + "15": 0.2808641975308642, + "16": 0.11504424778761062, + "17": 0.060240963855421686, + "18": 0.041935483870967745, + "19": 0.00906344410876133, + "20": 0.015015015015015015 + }, + "pass@64": { + "11": 0.9714285714285714, + "12": 0.8868501529051988, + "13": 0.7006172839506173, + "14": 0.5548961424332344, + "15": 0.3549382716049383, + "16": 0.13274336283185842, + "17": 0.06927710843373494, + "18": 0.04838709677419355, + "19": 0.00906344410876133, + "20": 0.015015015015015015 + }, + "pass@128": { + "11": 0.9746031746031746, + "12": 0.8929663608562691, + "13": 0.7160493827160493, + "14": 0.6320474777448071, + "15": 0.404320987654321, + "16": 0.14749262536873156, + "17": 0.08433734939759036, + "18": 0.06451612903225806, + "19": 0.012084592145015106, + "20": 0.018018018018018018 + } + }, + "total_pass_at_k": { + "pass@1": 0.19407090464547677, + "pass@2": 0.22249388753056235, + "pass@4": 0.2567237163814181, + "pass@8": 0.29492665036674814, + "pass@16": 0.32793398533007334, + "pass@32": 0.35146699266503667, + "pass@64": 0.37163814180929094, + "pass@128": 0.3921149144254279 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6506024096385542, + "pass@2": 0.6987951807228916, + "pass@4": 0.7801204819277109, + "pass@8": 0.8283132530120482, + "pass@16": 0.9006024096385542, + "pass@32": 0.9337349397590361, + "pass@64": 0.9578313253012049, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5454545454545454, + "pass@2": 0.5987460815047022, + "pass@4": 0.6614420062695925, + "pass@8": 0.7492163009404389, + "pass@16": 0.8181818181818182, + "pass@32": 0.8432601880877743, + "pass@64": 0.8557993730407524, + "pass@128": 0.8589341692789969 + }, + "13": { + "pass@1": 0.40896358543417366, + "pass@2": 0.49019607843137253, + "pass@4": 0.5686274509803921, + "pass@8": 0.6386554621848739, + "pass@16": 0.680672268907563, + "pass@32": 0.7394957983193278, + "pass@64": 0.7787114845938375, + "pass@128": 0.8151260504201681 + }, + "14": { + "pass@1": 0.22807017543859648, + "pass@2": 0.26900584795321636, + "pass@4": 0.30994152046783624, + "pass@8": 0.35964912280701755, + "pass@16": 0.40350877192982454, + "pass@32": 0.4649122807017544, + "pass@64": 0.5321637426900585, + "pass@128": 0.5877192982456141 + }, + "15": { + "pass@1": 0.12121212121212122, + "pass@2": 0.1484848484848485, + "pass@4": 0.19090909090909092, + "pass@8": 0.23636363636363636, + "pass@16": 0.27575757575757576, + "pass@32": 0.32727272727272727, + "pass@64": 0.36666666666666664, + "pass@128": 0.4090909090909091 + }, + "16": { + "pass@1": 0.05128205128205128, + "pass@2": 0.07977207977207977, + "pass@4": 0.09401709401709402, + "pass@8": 0.1111111111111111, + "pass@16": 0.1225071225071225, + "pass@32": 0.1396011396011396, + "pass@64": 0.1623931623931624, + "pass@128": 0.19658119658119658 + }, + "17": { + "pass@1": 0.021739130434782608, + "pass@2": 0.037267080745341616, + "pass@4": 0.046583850931677016, + "pass@8": 0.06521739130434782, + "pass@16": 0.06832298136645963, + "pass@32": 0.06832298136645963, + "pass@64": 0.07763975155279502, + "pass@128": 0.09937888198757763 + }, + "18": { + "pass@1": 0.008823529411764706, + "pass@2": 0.008823529411764706, + "pass@4": 0.011764705882352941, + "pass@8": 0.011764705882352941, + "pass@16": 0.023529411764705882, + "pass@32": 0.029411764705882353, + "pass@64": 0.03235294117647059, + "pass@128": 0.041176470588235294 + }, + "19": { + "pass@1": 0.002976190476190476, + "pass@2": 0.002976190476190476, + "pass@4": 0.008928571428571428, + "pass@8": 0.01488095238095238, + "pass@16": 0.01488095238095238, + "pass@32": 0.01488095238095238, + "pass@64": 0.017857142857142856, + "pass@128": 0.023809523809523808 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0029940119760479044, + "pass@4": 0.0029940119760479044, + "pass@8": 0.0029940119760479044, + "pass@16": 0.0029940119760479044, + "pass@32": 0.0029940119760479044, + "pass@64": 0.0029940119760479044, + "pass@128": 0.0029940119760479044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6506024096385542, + "12": 0.5454545454545454, + "13": 0.40896358543417366, + "14": 0.22807017543859648, + "15": 0.12121212121212122, + "16": 0.05128205128205128, + "17": 0.021739130434782608, + "18": 0.008823529411764706, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@2": { + "11": 0.6987951807228916, + "12": 0.5987460815047022, + "13": 0.49019607843137253, + "14": 0.26900584795321636, + "15": 0.1484848484848485, + "16": 0.07977207977207977, + "17": 0.037267080745341616, + "18": 0.008823529411764706, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@4": { + "11": 0.7801204819277109, + "12": 0.6614420062695925, + "13": 0.5686274509803921, + "14": 0.30994152046783624, + "15": 0.19090909090909092, + "16": 0.09401709401709402, + "17": 0.046583850931677016, + "18": 0.011764705882352941, + "19": 0.008928571428571428, + "20": 0.0029940119760479044 + }, + "pass@8": { + "11": 0.8283132530120482, + "12": 0.7492163009404389, + "13": 0.6386554621848739, + "14": 0.35964912280701755, + "15": 0.23636363636363636, + "16": 0.1111111111111111, + "17": 0.06521739130434782, + "18": 0.011764705882352941, + "19": 0.01488095238095238, + "20": 0.0029940119760479044 + }, + "pass@16": { + "11": 0.9006024096385542, + "12": 0.8181818181818182, + "13": 0.680672268907563, + "14": 0.40350877192982454, + "15": 0.27575757575757576, + "16": 0.1225071225071225, + "17": 0.06832298136645963, + "18": 0.023529411764705882, + "19": 0.01488095238095238, + "20": 0.0029940119760479044 + }, + "pass@32": { + "11": 0.9337349397590361, + "12": 0.8432601880877743, + "13": 0.7394957983193278, + "14": 0.4649122807017544, + "15": 0.32727272727272727, + "16": 0.1396011396011396, + "17": 0.06832298136645963, + "18": 0.029411764705882353, + "19": 0.01488095238095238, + "20": 0.0029940119760479044 + }, + "pass@64": { + "11": 0.9578313253012049, + "12": 0.8557993730407524, + "13": 0.7787114845938375, + "14": 0.5321637426900585, + "15": 0.36666666666666664, + "16": 0.1623931623931624, + "17": 0.07763975155279502, + "18": 0.03235294117647059, + "19": 0.017857142857142856, + "20": 0.0029940119760479044 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8589341692789969, + "13": 0.8151260504201681, + "14": 0.5877192982456141, + "15": 0.4090909090909091, + "16": 0.19658119658119658, + "17": 0.09937888198757763, + "18": 0.041176470588235294, + "19": 0.023809523809523808, + "20": 0.0029940119760479044 + } + }, + "total_pass_at_k": { + "pass@1": 0.2030924769550996, + "pass@2": 0.23312518584597086, + "pass@4": 0.26702349093071664, + "pass@8": 0.30121914956883733, + "pass@16": 0.3303597977995837, + "pass@32": 0.3559322033898305, + "pass@64": 0.37823371989295274, + "pass@128": 0.4005352363960749 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.607, + "pass@2": 0.669, + "pass@4": 0.75, + "pass@8": 0.832, + "pass@16": 0.906, + "pass@32": 0.938, + "pass@64": 0.957, + "pass@128": 0.963 + }, + "12": { + "pass@1": 0.572, + "pass@2": 0.625, + "pass@4": 0.685, + "pass@8": 0.76, + "pass@16": 0.821, + "pass@32": 0.851, + "pass@64": 0.864, + "pass@128": 0.873 + }, + "13": { + "pass@1": 0.415, + "pass@2": 0.477, + "pass@4": 0.556, + "pass@8": 0.619, + "pass@16": 0.67, + "pass@32": 0.715, + "pass@64": 0.752, + "pass@128": 0.777 + }, + "14": { + "pass@1": 0.235, + "pass@2": 0.296, + "pass@4": 0.348, + "pass@8": 0.413, + "pass@16": 0.473, + "pass@32": 0.528, + "pass@64": 0.584, + "pass@128": 0.64 + }, + "15": { + "pass@1": 0.121, + "pass@2": 0.154, + "pass@4": 0.194, + "pass@8": 0.232, + "pass@16": 0.284, + "pass@32": 0.334, + "pass@64": 0.393, + "pass@128": 0.438 + }, + "16": { + "pass@1": 0.046, + "pass@2": 0.075, + "pass@4": 0.088, + "pass@8": 0.105, + "pass@16": 0.12, + "pass@32": 0.139, + "pass@64": 0.161, + "pass@128": 0.185 + }, + "17": { + "pass@1": 0.025, + "pass@2": 0.036, + "pass@4": 0.047, + "pass@8": 0.057, + "pass@16": 0.067, + "pass@32": 0.082, + "pass@64": 0.094, + "pass@128": 0.115 + }, + "18": { + "pass@1": 0.01, + "pass@2": 0.011, + "pass@4": 0.014, + "pass@8": 0.019, + "pass@16": 0.028, + "pass@32": 0.036, + "pass@64": 0.041, + "pass@128": 0.052 + }, + "19": { + "pass@1": 0.002, + "pass@2": 0.004, + "pass@4": 0.007, + "pass@8": 0.011, + "pass@16": 0.012, + "pass@32": 0.013, + "pass@64": 0.015, + "pass@128": 0.018 + }, + "20": { + "pass@1": 0.001, + "pass@2": 0.002, + "pass@4": 0.003, + "pass@8": 0.005, + "pass@16": 0.006, + "pass@32": 0.007, + "pass@64": 0.007, + "pass@128": 0.008 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.607, + "12": 0.572, + "13": 0.415, + "14": 0.235, + "15": 0.121, + "16": 0.046, + "17": 0.025, + "18": 0.01, + "19": 0.002, + "20": 0.001 + }, + "pass@2": { + "11": 0.669, + "12": 0.625, + "13": 0.477, + "14": 0.296, + "15": 0.154, + "16": 0.075, + "17": 0.036, + "18": 0.011, + "19": 0.004, + "20": 0.002 + }, + "pass@4": { + "11": 0.75, + "12": 0.685, + "13": 0.556, + "14": 0.348, + "15": 0.194, + "16": 0.088, + "17": 0.047, + "18": 0.014, + "19": 0.007, + "20": 0.003 + }, + "pass@8": { + "11": 0.832, + "12": 0.76, + "13": 0.619, + "14": 0.413, + "15": 0.232, + "16": 0.105, + "17": 0.057, + "18": 0.019, + "19": 0.011, + "20": 0.005 + }, + "pass@16": { + "11": 0.906, + "12": 0.821, + "13": 0.67, + "14": 0.473, + "15": 0.284, + "16": 0.12, + "17": 0.067, + "18": 0.028, + "19": 0.012, + "20": 0.006 + }, + "pass@32": { + "11": 0.938, + "12": 0.851, + "13": 0.715, + "14": 0.528, + "15": 0.334, + "16": 0.139, + "17": 0.082, + "18": 0.036, + "19": 0.013, + "20": 0.007 + }, + "pass@64": { + "11": 0.957, + "12": 0.864, + "13": 0.752, + "14": 0.584, + "15": 0.393, + "16": 0.161, + "17": 0.094, + "18": 0.041, + "19": 0.015, + "20": 0.007 + }, + "pass@128": { + "11": 0.963, + "12": 0.873, + "13": 0.777, + "14": 0.64, + "15": 0.438, + "16": 0.185, + "17": 0.115, + "18": 0.052, + "19": 0.018, + "20": 0.008 + } + }, + "total_pass_at_k": { + "pass@1": 0.2034, + "pass@2": 0.2349, + "pass@4": 0.2692, + "pass@8": 0.3053, + "pass@16": 0.3387, + "pass@32": 0.3643, + "pass@64": 0.3868, + "pass@128": 0.4069 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M-ep4/composition/summary.csv b/Qwen3-100M-ep4/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9fd7a6ef8be6d5229a820a58d53cd15e4525e5e5 --- /dev/null +++ b/Qwen3-100M-ep4/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-85276,0.9949305555555555,0.44683984375,0.7064617598684211,0.0,0.0,0.0,200.0108559027778,327.42891796875,1152000,1280000,2432000 diff --git a/Qwen3-100M-ep4/composition/summary.json b/Qwen3-100M-ep4/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7cee1db64220448605dda054e269f6624112fdf0 --- /dev/null +++ b/Qwen3-100M-ep4/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-85276", + "id_acc": 0.9949305555555555, + "ood_acc": 0.44683984375, + "total_acc": 0.7064617598684211, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.0108559027778, + "ood_avg_resp_len": 327.42891796875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 576855, + "answer_accuracy": 0.7124058943250079, + "avg_response_len": 256.26435790784063, + "resp_tokens_sum": 207504426, + "pass_at_k": { + "pass@1": 0.7124058943250079, + "pass@2": 0.7482848133624225, + "pass@4": 0.7797965772724061, + "pass@8": 0.8060704977403156, + "pass@16": 0.8268832725059063, + "pass@32": 0.8434735001446694, + "pass@64": 0.8574080645300883, + "pass@128": 0.8695858362314258 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.964855272861357, + "pass@2": 0.9744083600445962, + "pass@4": 0.980863755415529, + "pass@8": 0.9845045622809016, + "pass@16": 0.9864817601267433, + "pass@32": 0.988192742885706, + "pass@64": 0.9911504413060571, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985367063492063, + "pass@2": 0.99933172415948, + "pass@4": 0.999864562762988, + "pass@8": 0.999995076184736, + "pass@16": 0.999999996268659, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997688609467456, + "pass@2": 0.9999836200670922, + "pass@4": 0.9999999417602385, + "pass@8": 0.9999999999999069, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999824719551282, + "pass@2": 0.9999976340096912, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.997337147887324, + "pass@2": 0.9977261700121991, + "pass@4": 0.9981745219347582, + "pass@8": 0.9988346121771267, + "pass@16": 0.9995479293349335, + "pass@32": 0.9999462946812021, + "pass@64": 0.9999998252073432, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9568431303116147, + "pass@2": 0.966704122147621, + "pass@4": 0.9737702782725814, + "pass@8": 0.9797293556840062, + "pass@16": 0.9839171023985908, + "pass@32": 0.9864784015256126, + "pass@64": 0.9885864447521222, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8533501059322034, + "pass@2": 0.8813041477601309, + "pass@4": 0.8969065471688916, + "pass@8": 0.9103012428913164, + "pass@16": 0.9232330989994582, + "pass@32": 0.9356599220900689, + "pass@64": 0.9473396438691338, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.760384012539185, + "pass@2": 0.8294336077308518, + "pass@4": 0.874541486977137, + "pass@8": 0.9027689973847379, + "pass@16": 0.9198466852847369, + "pass@32": 0.9308948171804944, + "pass@64": 0.9399058726440269, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.6034852024922118, + "pass@2": 0.6929789259695341, + "pass@4": 0.7630072742075453, + "pass@8": 0.8164932601872855, + "pass@16": 0.8526563967709537, + "pass@32": 0.8763087593659933, + "pass@64": 0.8936422032813724, + "pass@128": 0.9065420560747663 + }, + "15": { + "pass@1": 0.4252619219653179, + "pass@2": 0.5253370209594465, + "pass@4": 0.6099591160209015, + "pass@8": 0.6784789856366449, + "pass@16": 0.7314473595873143, + "pass@32": 0.7713284427298922, + "pass@64": 0.8031345386516523, + "pass@128": 0.8294797687861272 + }, + "16": { + "pass@1": 0.26829637096774195, + "pass@2": 0.3590876619253239, + "pass@4": 0.44438377874136703, + "pass@8": 0.5160205150620084, + "pass@16": 0.572072708861716, + "pass@32": 0.6173416875891375, + "pass@64": 0.6591309169128403, + "pass@128": 0.6967741935483871 + }, + "17": { + "pass@1": 0.210305274566474, + "pass@2": 0.2880021534067633, + "pass@4": 0.36638274885075767, + "pass@8": 0.4354115772958514, + "pass@16": 0.4924820483803051, + "pass@32": 0.5414044485040482, + "pass@64": 0.5843553236837202, + "pass@128": 0.6242774566473989 + }, + "18": { + "pass@1": 0.19066964285714286, + "pass@2": 0.2540649606299211, + "pass@4": 0.3153792696448659, + "pass@8": 0.3744374956178368, + "pass@16": 0.4304686242139373, + "pass@32": 0.4799023237679476, + "pass@64": 0.5191885486106175, + "pass@128": 0.5485714285714286 + }, + "19": { + "pass@1": 0.17203922672672672, + "pass@2": 0.2452760339079237, + "pass@4": 0.31713847114831356, + "pass@8": 0.3807029920118585, + "pass@16": 0.43294372077217796, + "pass@32": 0.47306262676523364, + "pass@64": 0.5035309613007347, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.16786317567567569, + "pass@2": 0.23649646201319421, + "pass@4": 0.3076818460755471, + "pass@8": 0.3678730399252492, + "pass@16": 0.41342134802722214, + "pass@32": 0.450621229184501, + "pass@64": 0.48393762875113866, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 578426, + "answer_accuracy": 0.704435405300078, + "avg_response_len": 263.05899381332813, + "resp_tokens_sum": 216003001, + "pass_at_k": { + "pass@1": 0.704435405300078, + "pass@2": 0.7394285546608842, + "pass@4": 0.7702634267559836, + "pass@8": 0.7961202574484137, + "pass@16": 0.8169219087902136, + "pass@32": 0.8337421048859038, + "pass@64": 0.8480070379164071, + "pass@128": 0.8607950116913484 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569833205521472, + "pass@2": 0.9659142372349164, + "pass@4": 0.9719421844700393, + "pass@8": 0.9748675318830776, + "pass@16": 0.9757861748653367, + "pass@32": 0.9762266888126843, + "pass@64": 0.9769938650304216, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998414855072464, + "pass@2": 0.9999967904827115, + "pass@4": 0.9999999997282949, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999776146131805, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948283450704225, + "pass@2": 0.9955154153266054, + "pass@4": 0.9962789690901313, + "pass@8": 0.9971273245981712, + "pass@16": 0.997797475275284, + "pass@32": 0.9984192527437615, + "pass@64": 0.9993013197287653, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957662799401198, + "pass@2": 0.9974428603423076, + "pass@4": 0.9988750589372435, + "pass@8": 0.9997585789040461, + "pass@16": 0.9999876161726882, + "pass@32": 0.9999999824517899, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9643731174698795, + "pass@2": 0.9750883455080163, + "pass@4": 0.9803072865138847, + "pass@8": 0.9823661486047269, + "pass@16": 0.9832962407555205, + "pass@32": 0.9844320989052803, + "pass@64": 0.9860781709515227, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8863636363636364, + "pass@2": 0.9133642287660749, + "pass@4": 0.9258904760729354, + "pass@8": 0.9354300779654707, + "pass@16": 0.945221377305145, + "pass@32": 0.9540520042583904, + "pass@64": 0.9598380142135106, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.6867997198879552, + "pass@2": 0.7681042397273864, + "pass@4": 0.824483724986057, + "pass@8": 0.8627893286369971, + "pass@16": 0.8897924787212615, + "pass@32": 0.9118141519741789, + "pass@64": 0.9305813041850224, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.5445906432748538, + "pass@2": 0.6282847970483952, + "pass@4": 0.6919824232497251, + "pass@8": 0.7401227434606403, + "pass@16": 0.7772650766591268, + "pass@32": 0.8067142511689581, + "pass@64": 0.8289431092869494, + "pass@128": 0.8450292397660819 + }, + "15": { + "pass@1": 0.3677556818181818, + "pass@2": 0.4612525351944643, + "pass@4": 0.5395015228210107, + "pass@8": 0.6038984673310392, + "pass@16": 0.6548475782041733, + "pass@32": 0.695211253240757, + "pass@64": 0.7307349783554408, + "pass@128": 0.7636363636363637 + }, + "16": { + "pass@1": 0.23595530626780628, + "pass@2": 0.30808173217129914, + "pass@4": 0.37879288913672104, + "pass@8": 0.44199327843430775, + "pass@16": 0.49717687261406224, + "pass@32": 0.5475518095537767, + "pass@64": 0.5956511172414417, + "pass@128": 0.6410256410256411 + }, + "17": { + "pass@1": 0.23090547360248448, + "pass@2": 0.31294512948109743, + "pass@4": 0.3923752244478753, + "pass@8": 0.4620048777355182, + "pass@16": 0.5184937076070995, + "pass@32": 0.5641996443460395, + "pass@64": 0.6058905683329744, + "pass@128": 0.6490683229813664 + }, + "18": { + "pass@1": 0.19181985294117648, + "pass@2": 0.2639264416396479, + "pass@4": 0.3402037715873749, + "pass@8": 0.41292313703410055, + "pass@16": 0.4755867515992984, + "pass@32": 0.5242663505584598, + "pass@64": 0.5607374408176437, + "pass@128": 0.5911764705882353 + }, + "19": { + "pass@1": 0.17740885416666666, + "pass@2": 0.24643025578833894, + "pass@4": 0.3146818307867766, + "pass@8": 0.37225657621985075, + "pass@16": 0.4180985090398595, + "pass@32": 0.4553241566350863, + "pass@64": 0.48681976016382017, + "pass@128": 0.5178571428571429 + }, + "20": { + "pass@1": 0.1394320733532934, + "pass@2": 0.2010787725729644, + "pass@4": 0.26803790703183067, + "pass@8": 0.32950827537338695, + "pass@16": 0.37733200724855503, + "pass@32": 0.4121459313058152, + "pass@64": 0.43980782684006975, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 562834, + "answer_accuracy": 0.7025308555679821, + "avg_response_len": 282.1113771169516, + "resp_tokens_sum": 226014094, + "pass_at_k": { + "pass@1": 0.7025308555679821, + "pass@2": 0.736340083665345, + "pass@4": 0.7661264757077474, + "pass@8": 0.791289508043559, + "pass@16": 0.8116318796222768, + "pass@32": 0.8281989841030529, + "pass@64": 0.8425176638526883, + "pass@128": 0.8555679821057677 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9654617537313432, + "pass@2": 0.974182483253026, + "pass@4": 0.9814832566638125, + "pass@8": 0.9865134381095033, + "pass@16": 0.990010567366735, + "pass@32": 0.9930920761226464, + "pass@64": 0.9962352974533647, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999080882352941, + "pass@2": 0.9999992762853173, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999258306962026, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954016516516516, + "pass@2": 0.9980015694828687, + "pass@4": 0.9994780266768457, + "pass@8": 0.9999541707109151, + "pass@16": 0.9999996450540588, + "pass@32": 0.9999999999945218, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.994599075562701, + "pass@2": 0.9959007962630074, + "pass@4": 0.9973834390074231, + "pass@8": 0.9989100635493507, + "pass@16": 0.9998018508583297, + "pass@32": 0.9999941260810753, + "pass@64": 0.9999999994727716, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9669642857142857, + "pass@2": 0.977667635295588, + "pass@4": 0.9834914713041819, + "pass@8": 0.987697919919454, + "pass@16": 0.9903756780268099, + "pass@32": 0.9920980050763724, + "pass@64": 0.9932587543558059, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8925602064220184, + "pass@2": 0.927709778106865, + "pass@4": 0.9466711294115757, + "pass@8": 0.9594735205095442, + "pass@16": 0.968631391961948, + "pass@32": 0.9747149407936225, + "pass@64": 0.9783565974863963, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6791087962962963, + "pass@2": 0.7509379252940609, + "pass@4": 0.8003636541960031, + "pass@8": 0.836590700158435, + "pass@16": 0.8634558161248805, + "pass@32": 0.8838359924574606, + "pass@64": 0.8998857884792347, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5279349035608308, + "pass@2": 0.6192675354564355, + "pass@4": 0.6896984853896232, + "pass@8": 0.7423055698396044, + "pass@16": 0.7843204109962838, + "pass@32": 0.8204913013870107, + "pass@64": 0.8477207957073809, + "pass@128": 0.8664688427299704 + }, + "15": { + "pass@1": 0.36140046296296297, + "pass@2": 0.4426244137017595, + "pass@4": 0.5132164547139938, + "pass@8": 0.5721977881442194, + "pass@16": 0.6197088824105395, + "pass@32": 0.658967059013711, + "pass@64": 0.6955853807708048, + "pass@128": 0.7345679012345679 + }, + "16": { + "pass@1": 0.21766685103244837, + "pass@2": 0.2878512385896452, + "pass@4": 0.35677422180634516, + "pass@8": 0.4192935319550409, + "pass@16": 0.4744116086421201, + "pass@32": 0.5235822279834264, + "pass@64": 0.5713460103429768, + "pass@128": 0.6194690265486725 + }, + "17": { + "pass@1": 0.19837161144578314, + "pass@2": 0.26883901254624787, + "pass@4": 0.33928608895650064, + "pass@8": 0.40176271614166426, + "pass@16": 0.4530815948293738, + "pass@32": 0.49619312239293656, + "pass@64": 0.536260897714176, + "pass@128": 0.5692771084337349 + }, + "18": { + "pass@1": 0.19788306451612903, + "pass@2": 0.2630076041402083, + "pass@4": 0.3281824866649735, + "pass@8": 0.38767325712032175, + "pass@16": 0.4352260441352987, + "pass@32": 0.47043974442309966, + "pass@64": 0.4992493451488665, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17987632175226587, + "pass@2": 0.247976856697671, + "pass@4": 0.3147488179347673, + "pass@8": 0.37247260884174876, + "pass@16": 0.41889090519709676, + "pass@32": 0.45572123675521775, + "pass@64": 0.4860676102567291, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.15899493243243243, + "pass@2": 0.2257040505071999, + "pass@4": 0.29573212638960666, + "pass@8": 0.3606103751183562, + "pass@16": 0.41446968279744983, + "pass@32": 0.457917316065741, + "pass@64": 0.49484972985591286, + "pass@128": 0.5285285285285285 + } + } + } + } + } +] \ No newline at end of file diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl b/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cec815fc513d8f230321e303b3e2757acfe8564d --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72b5c20ac734fd52011e03baa0922792ae458dfcbd90a7ea26ab05c31d2a740e +size 1789986458 diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_metrics.json b/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8aff7bb888f127cbb07cdbda5c2c81896d60c3 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-38646", + "id": { + "answer_accuracy": 0.9932248263888889, + "avg_response_len": 200.3281736111111, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9932248263888889, + "pass@2": 0.9952680036089231, + "pass@4": 0.9965955903532892, + "pass@8": 0.9974907332779658, + "pass@16": 0.9981196797512945, + "pass@32": 0.9985813699315721, + "pass@64": 0.9989574187426506, + "pass@128": 0.9993333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.95884375, + "pass@2": 0.9682652559055118, + "pass@4": 0.9756639486314218, + "pass@8": 0.9813791969954038, + "pass@16": 0.9855953958960963, + "pass@32": 0.9888302276998161, + "pass@64": 0.9917058153968907, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999390625, + "pass@2": 0.9997413877952757, + "pass@4": 0.9999365339332582, + "pass@8": 0.9999964696284563, + "pass@16": 0.9999999930573713, + "pass@32": 0.9999999999999976, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9945, + "pass@2": 0.9970660679133861, + "pass@4": 0.9983636825084363, + "pass@8": 0.9990063609804485, + "pass@16": 0.9993881074316475, + "pass@32": 0.999688172223201, + "pass@64": 0.9999404409407284, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991796875, + "pass@2": 0.9996582185039371, + "pass@4": 0.9999079231346082, + "pass@8": 0.9999925579902273, + "pass@16": 0.9999999657010183, + "pass@32": 0.9999999999998928, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997015625, + "pass@2": 0.9979917568897638, + "pass@4": 0.9985803902324709, + "pass@8": 0.9990458205281643, + "pass@16": 0.9994464582913845, + "pass@32": 0.999767693754703, + "pass@64": 0.9999711810355609, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9955234375, + "pass@2": 0.9983201279527558, + "pass@4": 0.9996303053055869, + "pass@8": 0.999974862902426, + "pass@16": 0.9999998996583422, + "pass@32": 0.9999999999997113, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.994578125, + "pass@2": 0.996369217519685, + "pass@4": 0.9972775294338206, + "pass@8": 0.9980213304765657, + "pass@16": 0.9986472977257898, + "pass@32": 0.9989462357068267, + "pass@64": 0.9989993313106743, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.95884375, + "2": 1.0, + "3": 0.999390625, + "4": 0.9999921875, + "5": 0.9945, + "6": 0.9991796875, + "7": 0.997015625, + "8": 0.9955234375, + "9": 0.994578125 + }, + "per_op_avg_response_len": { + "10": 281.7881875, + "2": 115.6349609375, + "3": 144.9606328125, + "4": 159.9038359375, + "5": 173.56409375, + "6": 199.1092890625, + "7": 224.44571875, + "8": 243.821984375, + "9": 259.724859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.7881875, + "2": 115.6349609375, + "3": 144.9606328125, + "4": 159.9038359375, + "5": 173.56409375, + "6": 199.1092890625, + "7": 224.44571875, + "8": 243.821984375, + "9": 259.724859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376776, + "answer_accuracy": 0.9941109422492401, + "avg_response_len": 190.80592494089834, + "resp_tokens_sum": 72316972, + "pass_at_k": { + "pass@1": 0.9941109422492401, + "pass@2": 0.9958522471925055, + "pass@4": 0.9969521343709948, + "pass@8": 0.997695466193822, + "pass@16": 0.9982193570151264, + "pass@32": 0.9985217552165557, + "pass@64": 0.9987341050585805, + "pass@128": 0.9989868287740629 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9643482669616519, + "pass@2": 0.9724997386941677, + "pass@4": 0.9787036025917114, + "pass@8": 0.9837184467724707, + "pass@16": 0.9876176547055514, + "pass@32": 0.990048743242238, + "pass@64": 0.9918928791392225, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998805428134556, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967757936507936, + "pass@2": 0.9988044463192103, + "pass@4": 0.9997930303354938, + "pass@8": 0.9999940728357846, + "pass@16": 0.999999996928095, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981521332335329, + "pass@2": 0.999078737210618, + "pass@4": 0.9997273029868273, + "pass@8": 0.9999777200738017, + "pass@16": 0.9999998973084382, + "pass@32": 0.999999999999679, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998959874260355, + "pass@2": 0.9998143607603783, + "pass@4": 0.9999912155026479, + "pass@8": 0.9999999836752375, + "pass@16": 0.9999999999999905, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960687099358975, + "pass@2": 0.9985512252675145, + "pass@4": 0.9996649123787412, + "pass@8": 0.9999730867256917, + "pass@16": 0.9999998586577866, + "pass@32": 0.9999999999993815, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9940801056338028, + "pass@2": 0.9950447765332151, + "pass@4": 0.9956578631896364, + "pass@8": 0.996375867480201, + "pass@16": 0.9969724240620078, + "pass@32": 0.9971729386969103, + "pass@64": 0.9971830959162251, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387835, + "answer_accuracy": 0.9927788130733946, + "avg_response_len": 198.29017345183487, + "resp_tokens_sum": 77463246, + "pass_at_k": { + "pass@1": 0.9927788130733946, + "pass@2": 0.9945622255575279, + "pass@4": 0.9957723347122567, + "pass@8": 0.9965841163949335, + "pass@16": 0.9972021928309309, + "pass@32": 0.9978155261463467, + "pass@64": 0.9984095685624238, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9476131134969326, + "pass@2": 0.9578519002705184, + "pass@4": 0.9660574474778076, + "pass@8": 0.9719991236070697, + "pass@16": 0.9763583249372905, + "pass@32": 0.980656765738378, + "pass@64": 0.9852951841811889, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9992185635886424, + "pass@4": 0.9998076788696866, + "pass@8": 0.9999893019044129, + "pass@16": 0.9999999789617312, + "pass@32": 0.9999999999999926, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9955842391304348, + "pass@2": 0.9970807657195024, + "pass@4": 0.9974464667460045, + "pass@8": 0.9977680565376104, + "pass@16": 0.9983124881128989, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994627507163324, + "pass@2": 0.9999027028856349, + "pass@4": 0.9999971459513118, + "pass@8": 0.9999999985260104, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984420158959537, + "pass@2": 0.9991658037868099, + "pass@4": 0.9997657894822396, + "pass@8": 0.999983097148883, + "pass@16": 0.9999999407613452, + "pass@32": 0.9999999999999076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9972711267605634, + "pass@2": 0.9988802401020296, + "pass@4": 0.9997421626874106, + "pass@8": 0.9999832931204404, + "pass@16": 0.9999999422581453, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965147829341318, + "pass@2": 0.9973931320430004, + "pass@4": 0.9978456220292824, + "pass@8": 0.998468082786387, + "pass@16": 0.9992530482905866, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379584, + "answer_accuracy": 0.9928021426180114, + "avg_response_len": 211.8498859641781, + "resp_tokens_sum": 80997838, + "pass_at_k": { + "pass@1": 0.9928021426180114, + "pass@2": 0.9954099819691105, + "pass@4": 0.9970833203097687, + "pass@8": 0.9982141281769159, + "pass@16": 0.9989583224371812, + "pass@32": 0.9994229750215191, + "pass@64": 0.9997385605466621, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9642024253731343, + "pass@2": 0.9741138059701494, + "pass@4": 0.9819364163434796, + "pass@8": 0.9881400872944226, + "pass@16": 0.9925379015562921, + "pass@32": 0.9955510391342886, + "pass@64": 0.997754923360079, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9912913602941177, + "pass@2": 0.9954405974988421, + "pass@4": 0.9979701380158362, + "pass@8": 0.9993477897991202, + "pass@16": 0.9999126470598457, + "pass@32": 0.999998726614258, + "pass@64": 0.999999999987784, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9933742088607594, + "pass@2": 0.9947567558307585, + "pass@4": 0.9957734057965857, + "pass@8": 0.9969989697228501, + "pass@16": 0.9982483505948174, + "pass@32": 0.9992648536542246, + "pass@64": 0.9999088007454459, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931493993993994, + "pass@2": 0.9975064877869996, + "pass@4": 0.999478633301468, + "pass@8": 0.9999675398986609, + "pass@16": 0.9999998926590724, + "pass@32": 0.9999999999998089, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930667202572347, + "pass@2": 0.9967814011190724, + "pass@4": 0.9985162387257863, + "pass@8": 0.9994198002264989, + "pass@16": 0.9999085821695214, + "pass@32": 0.999998609246854, + "pass@64": 0.9999999999866449, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.83598125, + "avg_response_len": 378.03591796875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.83598125, + "pass@2": 0.8769260949802976, + "pass@4": 0.9062144027652703, + "pass@8": 0.927088376859717, + "pass@16": 0.9418586434981504, + "pass@32": 0.9525318270976024, + "pass@64": 0.9609100985370623, + "pass@128": 0.9681 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.970671875, + "pass@2": 0.9794998769685037, + "pass@4": 0.9857863502999626, + "pass@8": 0.9900398112371814, + "pass@16": 0.9931495940187157, + "pass@32": 0.9953983463354901, + "pass@64": 0.9969441662809593, + "pass@128": 0.998 + }, + "12": { + "pass@1": 0.966296875, + "pass@2": 0.9809158464566927, + "pass@4": 0.9901372777465317, + "pass@8": 0.9945470208837859, + "pass@16": 0.996475576710408, + "pass@32": 0.9975372922449343, + "pass@64": 0.9982497123716464, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.9553671875, + "pass@2": 0.973431348425197, + "pass@4": 0.9843508790776158, + "pass@8": 0.9907141222601963, + "pass@16": 0.9945169364443083, + "pass@32": 0.99655956862069, + "pass@64": 0.9974491256090368, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.949578125, + "pass@2": 0.9686524360236218, + "pass@4": 0.9809611437007878, + "pass@8": 0.9883708817653398, + "pass@16": 0.9925049124837143, + "pass@32": 0.9948066901098008, + "pass@64": 0.9961112856236247, + "pass@128": 0.997 + }, + "15": { + "pass@1": 0.965375, + "pass@2": 0.9803630659448824, + "pass@4": 0.988369140607424, + "pass@8": 0.9930486759883792, + "pass@16": 0.9955475560142018, + "pass@32": 0.9967154674849088, + "pass@64": 0.997376405398854, + "pass@128": 0.998 + }, + "16": { + "pass@1": 0.887765625, + "pass@2": 0.9196377952755908, + "pass@4": 0.9421223665166858, + "pass@8": 0.9573031508529518, + "pass@16": 0.9670828002231159, + "pass@32": 0.9741994820696727, + "pass@64": 0.9803724490952007, + "pass@128": 0.986 + }, + "17": { + "pass@1": 0.856984375, + "pass@2": 0.8984932332677166, + "pass@4": 0.924790269216348, + "pass@8": 0.9417427006423036, + "pass@16": 0.9525526931446574, + "pass@32": 0.9599937953920628, + "pass@64": 0.9663307158630652, + "pass@128": 0.972 + }, + "18": { + "pass@1": 0.77284375, + "pass@2": 0.8381781496062998, + "pass@4": 0.8801090434008246, + "pass@8": 0.9078199779722668, + "pass@16": 0.9261821010586693, + "pass@32": 0.9385224656210279, + "pass@64": 0.9478627592988191, + "pass@128": 0.957 + }, + "19": { + "pass@1": 0.584125, + "pass@2": 0.6814692421259851, + "pass@4": 0.7541173871391076, + "pass@8": 0.8081890874701764, + "pass@16": 0.8491870017860157, + "pass@32": 0.8800031510922511, + "pass@64": 0.9043406794110973, + "pass@128": 0.924 + }, + "20": { + "pass@1": 0.4508046875, + "pass@2": 0.5486199557086607, + "pass@4": 0.6314001699475059, + "pass@8": 0.6991083395246123, + "pass@16": 0.7513872630976118, + "pass@32": 0.7915820120051243, + "pass@64": 0.8240636864182797, + "pass@128": 0.852 + } + }, + "per_op_accuracy": { + "11": 0.970671875, + "12": 0.966296875, + "13": 0.9553671875, + "14": 0.949578125, + "15": 0.965375, + "16": 0.887765625, + "17": 0.856984375, + "18": 0.77284375, + "19": 0.584125, + "20": 0.4508046875 + }, + "per_op_avg_response_len": { + "11": 301.01784375, + "12": 313.407609375, + "13": 334.547984375, + "14": 361.579859375, + "15": 381.650609375, + "16": 390.577, + "17": 411.0501640625, + "18": 426.7855859375, + "19": 424.2508671875, + "20": 435.49165625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 301.01784375, + "12": 313.407609375, + "13": 334.547984375, + "14": 361.579859375, + "15": 381.650609375, + "16": 390.577, + "17": 411.0501640625, + "18": 426.7855859375, + "19": 424.2508671875, + "20": 435.49165625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 359051, + "answer_accuracy": 0.8336065193164933, + "avg_response_len": 360.013558692422, + "resp_tokens_sum": 155065040, + "pass_at_k": { + "pass@1": 0.8336065193164933, + "pass@2": 0.8731902853014514, + "pass@4": 0.9019876336223236, + "pass@8": 0.9227793245050241, + "pass@16": 0.9379346127968918, + "pass@32": 0.9493515835193701, + "pass@64": 0.9586937969666259, + "pass@128": 0.9667161961367013 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9711623583569405, + "pass@2": 0.9805191998840088, + "pass@4": 0.986303336286222, + "pass@8": 0.989850641477996, + "pass@16": 0.99240868699634, + "pass@32": 0.994293234140082, + "pass@64": 0.9956686613210934, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9658368644067796, + "pass@2": 0.979869405889942, + "pass@4": 0.9899045508612269, + "pass@8": 0.9949535165900368, + "pass@16": 0.9969359713595375, + "pass@32": 0.9982218320963844, + "pass@64": 0.9992952648675736, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9498187695924765, + "pass@2": 0.9693292893639082, + "pass@4": 0.9815207164543299, + "pass@8": 0.9885514952344922, + "pass@16": 0.993030758542598, + "pass@32": 0.9957866897580077, + "pass@64": 0.9968002718717178, + "pass@128": 0.9968652037617555 + }, + "14": { + "pass@1": 0.9244548286604362, + "pass@2": 0.9464950971373904, + "pass@4": 0.962845547343965, + "pass@8": 0.9744775229835171, + "pass@16": 0.9831241464032126, + "pass@32": 0.9892894037314296, + "pass@64": 0.9925585252418944, + "pass@128": 0.9937694704049844 + }, + "15": { + "pass@1": 0.9763367052023122, + "pass@2": 0.9890843327568163, + "pass@4": 0.995195050131017, + "pass@8": 0.9987062171803248, + "pass@16": 0.9998947926322176, + "pass@32": 0.9999995163740522, + "pass@64": 0.9999999999997273, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8670362903225807, + "pass@2": 0.9042973710947418, + "pass@4": 0.9324377792493678, + "pass@8": 0.9511663793058849, + "pass@16": 0.9632457765868611, + "pass@32": 0.972380137786845, + "pass@64": 0.9803184368887754, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.8796739523121387, + "pass@2": 0.9130583354194169, + "pass@4": 0.9346565702950422, + "pass@8": 0.949022076996675, + "pass@16": 0.9577801252538675, + "pass@32": 0.9642535207391926, + "pass@64": 0.9711417638827387, + "pass@128": 0.976878612716763 + }, + "18": { + "pass@1": 0.7535491071428572, + "pass@2": 0.8142751687289084, + "pass@4": 0.854199288124698, + "pass@8": 0.8827058936508151, + "pass@16": 0.9034225204407647, + "pass@32": 0.9183155907983432, + "pass@64": 0.9304605804851537, + "pass@128": 0.9428571428571428 + }, + "19": { + "pass@1": 0.5902073948948949, + "pass@2": 0.6823130955758907, + "pass@4": 0.7493216109810598, + "pass@8": 0.7968110404034479, + "pass@16": 0.8325350334716398, + "pass@32": 0.859961150550141, + "pass@64": 0.8827473329655091, + "pass@128": 0.9009009009009009 + }, + "20": { + "pass@1": 0.4485735735735736, + "pass@2": 0.5452467723629141, + "pass@4": 0.6280490980294128, + "pass@8": 0.6976146762314849, + "pass@16": 0.7542370813489142, + "pass@32": 0.799241057750153, + "pass@64": 0.8368248092112288, + "pass@128": 0.8708708708708709 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 363264, + "answer_accuracy": 0.8438893844781445, + "avg_response_len": 373.32513287986916, + "resp_tokens_sum": 160703030, + "pass_at_k": { + "pass@1": 0.8438893844781445, + "pass@2": 0.8844832296693336, + "pass@4": 0.9135616027200477, + "pass@8": 0.9341262715675477, + "pass@16": 0.9485360735641319, + "pass@32": 0.9589141991748595, + "pass@64": 0.9667030192282954, + "pass@128": 0.9726434730895034 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9768213478915663, + "pass@2": 0.982426507209942, + "pass@4": 0.9875867705919291, + "pass@8": 0.9916699109297928, + "pass@16": 0.9941198985224184, + "pass@32": 0.9952745122110707, + "pass@64": 0.9962408632343975, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.965492750783699, + "pass@2": 0.9797927517093281, + "pass@4": 0.9877445484870819, + "pass@8": 0.9923678770900078, + "pass@16": 0.9955227193860775, + "pass@32": 0.9973880317416322, + "pass@64": 0.9984300583339356, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9623599439775911, + "pass@2": 0.9779697804318578, + "pass@4": 0.9872410336838148, + "pass@8": 0.9920194344744313, + "pass@16": 0.9942491977157041, + "pass@32": 0.9950794358980206, + "pass@64": 0.9957983155432268, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.9557291666666666, + "pass@2": 0.9763930618870013, + "pass@4": 0.9898799393496869, + "pass@8": 0.9971921128374414, + "pass@16": 0.9996866455629769, + "pass@32": 0.9999957420781709, + "pass@64": 0.9999999999150341, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9709043560606061, + "pass@2": 0.9843362264376043, + "pass@4": 0.9913319070911589, + "pass@8": 0.9957115636967746, + "pass@16": 0.9984277498659098, + "pass@32": 0.9997408383719552, + "pass@64": 0.9999955357422065, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9107015669515669, + "pass@2": 0.9377124133521775, + "pass@4": 0.9564160948898481, + "pass@8": 0.9692630161138946, + "pass@16": 0.9776624723599159, + "pass@32": 0.9832541934776856, + "pass@64": 0.9874615955124632, + "pass@128": 0.9914529914529915 + }, + "17": { + "pass@1": 0.8672360248447205, + "pass@2": 0.9112839750330124, + "pass@4": 0.9387636921937549, + "pass@8": 0.9558194693071742, + "pass@16": 0.9657225480978907, + "pass@32": 0.9716417279156446, + "pass@64": 0.9759516660247486, + "pass@128": 0.9782608695652174 + }, + "18": { + "pass@1": 0.7839613970588235, + "pass@2": 0.8523347035664656, + "pass@4": 0.8947464600013231, + "pass@8": 0.9214377697528552, + "pass@16": 0.9388117402083257, + "pass@32": 0.951005420422579, + "pass@64": 0.9607119551856902, + "pass@128": 0.9705882352941176 + }, + "19": { + "pass@1": 0.5933547247023809, + "pass@2": 0.6915281830005624, + "pass@4": 0.766301561356169, + "pass@8": 0.8222817559953494, + "pass@16": 0.8649021620408612, + "pass@32": 0.8975421565193381, + "pass@64": 0.9221817078073432, + "pass@128": 0.9404761904761905 + }, + "20": { + "pass@1": 0.4492889221556886, + "pass@2": 0.5484648321467304, + "pass@4": 0.6333525578202419, + "pass@8": 0.7015890174857582, + "pass@16": 0.7546736568184237, + "pass@32": 0.796908044139665, + "pass@64": 0.829107778372377, + "pass@128": 0.8502994011976048 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 347741, + "answer_accuracy": 0.8302954041870416, + "avg_response_len": 401.41232665418704, + "resp_tokens_sum": 168117905, + "pass_at_k": { + "pass@1": 0.8302954041870416, + "pass@2": 0.8730007757902922, + "pass@4": 0.9030097709554368, + "pass@8": 0.9242862745587226, + "pass@16": 0.9390310658691144, + "pass@32": 0.9492425857604578, + "pass@64": 0.9572353621372555, + "pass@128": 0.9648533007334963 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.963640873015873, + "pass@2": 0.9752730127484065, + "pass@4": 0.9833094119187484, + "pass@8": 0.9885337281484384, + "pass@16": 0.9929572101573484, + "pass@32": 0.9967672908887808, + "pass@64": 0.9991148007644227, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9675793195718655, + "pass@2": 0.9831443087962632, + "pass@4": 0.9927234121537561, + "pass@8": 0.9962327927192676, + "pass@16": 0.9969067136543522, + "pass@32": 0.9969418396245668, + "pass@64": 0.9969418960244639, + "pass@128": 0.9969418960244648 + }, + "13": { + "pass@1": 0.953125, + "pass@2": 0.9724694243948675, + "pass@4": 0.9839528441467964, + "pass@8": 0.9914051147315478, + "pass@16": 0.9962751879157801, + "pass@32": 0.9989514073218897, + "pass@64": 0.9999069142994649, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9672663204747775, + "pass@2": 0.9819023224841703, + "pass@4": 0.9891655304029129, + "pass@8": 0.9926525053330164, + "pass@16": 0.9941520139636356, + "pass@32": 0.9947959873034944, + "pass@64": 0.9955489586054439, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9480372299382716, + "pass@2": 0.9670028768105379, + "pass@4": 0.9780621108704005, + "pass@8": 0.9842947803211456, + "pass@16": 0.9879716058879758, + "pass@32": 0.9901270313479683, + "pass@64": 0.9919070327284573, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.882973820058997, + "pass@2": 0.9149514842171274, + "pass@4": 0.9361787777080962, + "pass@8": 0.9505317245196182, + "pass@16": 0.95963740909398, + "pass@32": 0.9664879570651409, + "pass@64": 0.9730817511351202, + "pass@128": 0.9793510324483776 + }, + "17": { + "pass@1": 0.8233951430722891, + "pass@2": 0.8709084616023144, + "pass@4": 0.900955382553837, + "pass@8": 0.920503593025735, + "pass@16": 0.9343316545762004, + "pass@32": 0.9442573518899536, + "pass@64": 0.9519856298182777, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.7824344758064516, + "pass@2": 0.8496388430276858, + "pass@4": 0.8933080521184847, + "pass@8": 0.9212389467048724, + "pass@16": 0.9380265394631339, + "pass@32": 0.9476456964449388, + "pass@64": 0.9534177140834851, + "pass@128": 0.9580645161290322 + }, + "19": { + "pass@1": 0.5686367069486404, + "pass@2": 0.6704094012417636, + "pass@4": 0.7465739156094909, + "pass@8": 0.8053303353516325, + "pass@16": 0.8499870368405819, + "pass@32": 0.8823623062494177, + "pass@64": 0.907953968913336, + "pass@128": 0.9305135951661632 + }, + "20": { + "pass@1": 0.4545561186186186, + "pass@2": 0.5521487284528623, + "pass@4": 0.6327929909662195, + "pass@8": 0.6981138753732286, + "pass@16": 0.7452411820752879, + "pass@32": 0.7785809400350618, + "pass@64": 0.8062433242479462, + "pass@128": 0.8348348348348348 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9104650493421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9104650493421053, + "pass@2": 0.9329827885412267, + "pass@4": 0.9490265442543343, + "pass@8": 0.960436861478886, + "pass@16": 0.968508608039099, + "pass@32": 0.9743447684400129, + "pass@64": 0.9789325133712852, + "pass@128": 0.9828947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.95884375, + "pass@2": 0.9682652559055118, + "pass@4": 0.9756639486314218, + "pass@8": 0.9813791969954038, + "pass@16": 0.9855953958960963, + "pass@32": 0.9888302276998161, + "pass@64": 0.9917058153968907, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999390625, + "pass@2": 0.9997413877952757, + "pass@4": 0.9999365339332582, + "pass@8": 0.9999964696284563, + "pass@16": 0.9999999930573713, + "pass@32": 0.9999999999999976, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9945, + "pass@2": 0.9970660679133861, + "pass@4": 0.9983636825084363, + "pass@8": 0.9990063609804485, + "pass@16": 0.9993881074316475, + "pass@32": 0.999688172223201, + "pass@64": 0.9999404409407284, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991796875, + "pass@2": 0.9996582185039371, + "pass@4": 0.9999079231346082, + "pass@8": 0.9999925579902273, + "pass@16": 0.9999999657010183, + "pass@32": 0.9999999999998928, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997015625, + "pass@2": 0.9979917568897638, + "pass@4": 0.9985803902324709, + "pass@8": 0.9990458205281643, + "pass@16": 0.9994464582913845, + "pass@32": 0.999767693754703, + "pass@64": 0.9999711810355609, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9955234375, + "pass@2": 0.9983201279527558, + "pass@4": 0.9996303053055869, + "pass@8": 0.999974862902426, + "pass@16": 0.9999998996583422, + "pass@32": 0.9999999999997113, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.994578125, + "pass@2": 0.996369217519685, + "pass@4": 0.9972775294338206, + "pass@8": 0.9980213304765657, + "pass@16": 0.9986472977257898, + "pass@32": 0.9989462357068267, + "pass@64": 0.9989993313106743, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.970671875, + "pass@2": 0.9794998769685037, + "pass@4": 0.9857863502999626, + "pass@8": 0.9900398112371814, + "pass@16": 0.9931495940187157, + "pass@32": 0.9953983463354901, + "pass@64": 0.9969441662809593, + "pass@128": 0.998 + }, + "12": { + "pass@1": 0.966296875, + "pass@2": 0.9809158464566927, + "pass@4": 0.9901372777465317, + "pass@8": 0.9945470208837859, + "pass@16": 0.996475576710408, + "pass@32": 0.9975372922449343, + "pass@64": 0.9982497123716464, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.9553671875, + "pass@2": 0.973431348425197, + "pass@4": 0.9843508790776158, + "pass@8": 0.9907141222601963, + "pass@16": 0.9945169364443083, + "pass@32": 0.99655956862069, + "pass@64": 0.9974491256090368, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.949578125, + "pass@2": 0.9686524360236218, + "pass@4": 0.9809611437007878, + "pass@8": 0.9883708817653398, + "pass@16": 0.9925049124837143, + "pass@32": 0.9948066901098008, + "pass@64": 0.9961112856236247, + "pass@128": 0.997 + }, + "15": { + "pass@1": 0.965375, + "pass@2": 0.9803630659448824, + "pass@4": 0.988369140607424, + "pass@8": 0.9930486759883792, + "pass@16": 0.9955475560142018, + "pass@32": 0.9967154674849088, + "pass@64": 0.997376405398854, + "pass@128": 0.998 + }, + "16": { + "pass@1": 0.887765625, + "pass@2": 0.9196377952755908, + "pass@4": 0.9421223665166858, + "pass@8": 0.9573031508529518, + "pass@16": 0.9670828002231159, + "pass@32": 0.9741994820696727, + "pass@64": 0.9803724490952007, + "pass@128": 0.986 + }, + "17": { + "pass@1": 0.856984375, + "pass@2": 0.8984932332677166, + "pass@4": 0.924790269216348, + "pass@8": 0.9417427006423036, + "pass@16": 0.9525526931446574, + "pass@32": 0.9599937953920628, + "pass@64": 0.9663307158630652, + "pass@128": 0.972 + }, + "18": { + "pass@1": 0.77284375, + "pass@2": 0.8381781496062998, + "pass@4": 0.8801090434008246, + "pass@8": 0.9078199779722668, + "pass@16": 0.9261821010586693, + "pass@32": 0.9385224656210279, + "pass@64": 0.9478627592988191, + "pass@128": 0.957 + }, + "19": { + "pass@1": 0.584125, + "pass@2": 0.6814692421259851, + "pass@4": 0.7541173871391076, + "pass@8": 0.8081890874701764, + "pass@16": 0.8491870017860157, + "pass@32": 0.8800031510922511, + "pass@64": 0.9043406794110973, + "pass@128": 0.924 + }, + "20": { + "pass@1": 0.4508046875, + "pass@2": 0.5486199557086607, + "pass@4": 0.6314001699475059, + "pass@8": 0.6991083395246123, + "pass@16": 0.7513872630976118, + "pass@32": 0.7915820120051243, + "pass@64": 0.8240636864182797, + "pass@128": 0.852 + } + }, + "per_op_accuracy": { + "10": 0.95884375, + "2": 1.0, + "3": 0.999390625, + "4": 0.9999921875, + "5": 0.9945, + "6": 0.9991796875, + "7": 0.997015625, + "8": 0.9955234375, + "9": 0.994578125, + "11": 0.970671875, + "12": 0.966296875, + "13": 0.9553671875, + "14": 0.949578125, + "15": 0.965375, + "16": 0.887765625, + "17": 0.856984375, + "18": 0.77284375, + "19": 0.584125, + "20": 0.4508046875 + }, + "per_op_avg_response_len": { + "10": 281.7881875, + "2": 115.6349609375, + "3": 144.9606328125, + "4": 159.9038359375, + "5": 173.56409375, + "6": 199.1092890625, + "7": 224.44571875, + "8": 243.821984375, + "9": 259.724859375, + "11": 301.01784375, + "12": 313.407609375, + "13": 334.547984375, + "14": 361.579859375, + "15": 381.650609375, + "16": 390.577, + "17": 411.0501640625, + "18": 426.7855859375, + "19": 424.2508671875, + "20": 435.49165625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.7881875, + "2": 115.6349609375, + "3": 144.9606328125, + "4": 159.9038359375, + "5": 173.56409375, + "6": 199.1092890625, + "7": 224.44571875, + "8": 243.821984375, + "9": 259.724859375, + "11": 301.01784375, + "12": 313.407609375, + "13": 334.547984375, + "14": 361.579859375, + "15": 381.650609375, + "16": 390.577, + "17": 411.0501640625, + "18": 426.7855859375, + "19": 424.2508671875, + "20": 435.49165625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 735827, + "answer_accuracy": 0.9087335500316156, + "avg_response_len": 280.81283097533986, + "resp_tokens_sum": 227382012, + "pass_at_k": { + "pass@1": 0.9087335500316156, + "pass@2": 0.9306044600025811, + "pass@4": 0.9464375050603238, + "pass@8": 0.9578451948086137, + "pass@16": 0.9661519899120048, + "pass@32": 0.9723665816849368, + "pass@64": 0.9774353954902267, + "pass@128": 0.9818210559595321 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9643482669616519, + "pass@2": 0.9724997386941677, + "pass@4": 0.9787036025917114, + "pass@8": 0.9837184467724707, + "pass@16": 0.9876176547055514, + "pass@32": 0.990048743242238, + "pass@64": 0.9918928791392225, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998805428134556, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967757936507936, + "pass@2": 0.9988044463192103, + "pass@4": 0.9997930303354938, + "pass@8": 0.9999940728357846, + "pass@16": 0.999999996928095, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981521332335329, + "pass@2": 0.999078737210618, + "pass@4": 0.9997273029868273, + "pass@8": 0.9999777200738017, + "pass@16": 0.9999998973084382, + "pass@32": 0.999999999999679, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998959874260355, + "pass@2": 0.9998143607603783, + "pass@4": 0.9999912155026479, + "pass@8": 0.9999999836752375, + "pass@16": 0.9999999999999905, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960687099358975, + "pass@2": 0.9985512252675145, + "pass@4": 0.9996649123787412, + "pass@8": 0.9999730867256917, + "pass@16": 0.9999998586577866, + "pass@32": 0.9999999999993815, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9940801056338028, + "pass@2": 0.9950447765332151, + "pass@4": 0.9956578631896364, + "pass@8": 0.996375867480201, + "pass@16": 0.9969724240620078, + "pass@32": 0.9971729386969103, + "pass@64": 0.9971830959162251, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9711623583569405, + "pass@2": 0.9805191998840088, + "pass@4": 0.986303336286222, + "pass@8": 0.989850641477996, + "pass@16": 0.99240868699634, + "pass@32": 0.994293234140082, + "pass@64": 0.9956686613210934, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9658368644067796, + "pass@2": 0.979869405889942, + "pass@4": 0.9899045508612269, + "pass@8": 0.9949535165900368, + "pass@16": 0.9969359713595375, + "pass@32": 0.9982218320963844, + "pass@64": 0.9992952648675736, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9498187695924765, + "pass@2": 0.9693292893639082, + "pass@4": 0.9815207164543299, + "pass@8": 0.9885514952344922, + "pass@16": 0.993030758542598, + "pass@32": 0.9957866897580077, + "pass@64": 0.9968002718717178, + "pass@128": 0.9968652037617555 + }, + "14": { + "pass@1": 0.9244548286604362, + "pass@2": 0.9464950971373904, + "pass@4": 0.962845547343965, + "pass@8": 0.9744775229835171, + "pass@16": 0.9831241464032126, + "pass@32": 0.9892894037314296, + "pass@64": 0.9925585252418944, + "pass@128": 0.9937694704049844 + }, + "15": { + "pass@1": 0.9763367052023122, + "pass@2": 0.9890843327568163, + "pass@4": 0.995195050131017, + "pass@8": 0.9987062171803248, + "pass@16": 0.9998947926322176, + "pass@32": 0.9999995163740522, + "pass@64": 0.9999999999997273, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8670362903225807, + "pass@2": 0.9042973710947418, + "pass@4": 0.9324377792493678, + "pass@8": 0.9511663793058849, + "pass@16": 0.9632457765868611, + "pass@32": 0.972380137786845, + "pass@64": 0.9803184368887754, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.8796739523121387, + "pass@2": 0.9130583354194169, + "pass@4": 0.9346565702950422, + "pass@8": 0.949022076996675, + "pass@16": 0.9577801252538675, + "pass@32": 0.9642535207391926, + "pass@64": 0.9711417638827387, + "pass@128": 0.976878612716763 + }, + "18": { + "pass@1": 0.7535491071428572, + "pass@2": 0.8142751687289084, + "pass@4": 0.854199288124698, + "pass@8": 0.8827058936508151, + "pass@16": 0.9034225204407647, + "pass@32": 0.9183155907983432, + "pass@64": 0.9304605804851537, + "pass@128": 0.9428571428571428 + }, + "19": { + "pass@1": 0.5902073948948949, + "pass@2": 0.6823130955758907, + "pass@4": 0.7493216109810598, + "pass@8": 0.7968110404034479, + "pass@16": 0.8325350334716398, + "pass@32": 0.859961150550141, + "pass@64": 0.8827473329655091, + "pass@128": 0.9009009009009009 + }, + "20": { + "pass@1": 0.4485735735735736, + "pass@2": 0.5452467723629141, + "pass@4": 0.6280490980294128, + "pass@8": 0.6976146762314849, + "pass@16": 0.7542370813489142, + "pass@32": 0.799241057750153, + "pass@64": 0.8368248092112288, + "pass@128": 0.8708708708708709 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 751099, + "answer_accuracy": 0.9147250097427904, + "avg_response_len": 290.0505114964926, + "resp_tokens_sum": 238166276, + "pass_at_k": { + "pass@1": 0.9147250097427904, + "pass@2": 0.9368544058892373, + "pass@4": 0.9526741754464987, + "pass@8": 0.9638412119281327, + "pass@16": 0.9716894634319847, + "pass@32": 0.9774218920691676, + "pass@64": 0.9817877251624777, + "pass@128": 0.9851909586905689 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9476131134969326, + "pass@2": 0.9578519002705184, + "pass@4": 0.9660574474778076, + "pass@8": 0.9719991236070697, + "pass@16": 0.9763583249372905, + "pass@32": 0.980656765738378, + "pass@64": 0.9852951841811889, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9992185635886424, + "pass@4": 0.9998076788696866, + "pass@8": 0.9999893019044129, + "pass@16": 0.9999999789617312, + "pass@32": 0.9999999999999926, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9955842391304348, + "pass@2": 0.9970807657195024, + "pass@4": 0.9974464667460045, + "pass@8": 0.9977680565376104, + "pass@16": 0.9983124881128989, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994627507163324, + "pass@2": 0.9999027028856349, + "pass@4": 0.9999971459513118, + "pass@8": 0.9999999985260104, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984420158959537, + "pass@2": 0.9991658037868099, + "pass@4": 0.9997657894822396, + "pass@8": 0.999983097148883, + "pass@16": 0.9999999407613452, + "pass@32": 0.9999999999999076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9972711267605634, + "pass@2": 0.9988802401020296, + "pass@4": 0.9997421626874106, + "pass@8": 0.9999832931204404, + "pass@16": 0.9999999422581453, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965147829341318, + "pass@2": 0.9973931320430004, + "pass@4": 0.9978456220292824, + "pass@8": 0.998468082786387, + "pass@16": 0.9992530482905866, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9768213478915663, + "pass@2": 0.982426507209942, + "pass@4": 0.9875867705919291, + "pass@8": 0.9916699109297928, + "pass@16": 0.9941198985224184, + "pass@32": 0.9952745122110707, + "pass@64": 0.9962408632343975, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.965492750783699, + "pass@2": 0.9797927517093281, + "pass@4": 0.9877445484870819, + "pass@8": 0.9923678770900078, + "pass@16": 0.9955227193860775, + "pass@32": 0.9973880317416322, + "pass@64": 0.9984300583339356, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9623599439775911, + "pass@2": 0.9779697804318578, + "pass@4": 0.9872410336838148, + "pass@8": 0.9920194344744313, + "pass@16": 0.9942491977157041, + "pass@32": 0.9950794358980206, + "pass@64": 0.9957983155432268, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.9557291666666666, + "pass@2": 0.9763930618870013, + "pass@4": 0.9898799393496869, + "pass@8": 0.9971921128374414, + "pass@16": 0.9996866455629769, + "pass@32": 0.9999957420781709, + "pass@64": 0.9999999999150341, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9709043560606061, + "pass@2": 0.9843362264376043, + "pass@4": 0.9913319070911589, + "pass@8": 0.9957115636967746, + "pass@16": 0.9984277498659098, + "pass@32": 0.9997408383719552, + "pass@64": 0.9999955357422065, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9107015669515669, + "pass@2": 0.9377124133521775, + "pass@4": 0.9564160948898481, + "pass@8": 0.9692630161138946, + "pass@16": 0.9776624723599159, + "pass@32": 0.9832541934776856, + "pass@64": 0.9874615955124632, + "pass@128": 0.9914529914529915 + }, + "17": { + "pass@1": 0.8672360248447205, + "pass@2": 0.9112839750330124, + "pass@4": 0.9387636921937549, + "pass@8": 0.9558194693071742, + "pass@16": 0.9657225480978907, + "pass@32": 0.9716417279156446, + "pass@64": 0.9759516660247486, + "pass@128": 0.9782608695652174 + }, + "18": { + "pass@1": 0.7839613970588235, + "pass@2": 0.8523347035664656, + "pass@4": 0.8947464600013231, + "pass@8": 0.9214377697528552, + "pass@16": 0.9388117402083257, + "pass@32": 0.951005420422579, + "pass@64": 0.9607119551856902, + "pass@128": 0.9705882352941176 + }, + "19": { + "pass@1": 0.5933547247023809, + "pass@2": 0.6915281830005624, + "pass@4": 0.766301561356169, + "pass@8": 0.8222817559953494, + "pass@16": 0.8649021620408612, + "pass@32": 0.8975421565193381, + "pass@64": 0.9221817078073432, + "pass@128": 0.9404761904761905 + }, + "20": { + "pass@1": 0.4492889221556886, + "pass@2": 0.5484648321467304, + "pass@4": 0.6333525578202419, + "pass@8": 0.7015890174857582, + "pass@16": 0.7546736568184237, + "pass@32": 0.796908044139665, + "pass@64": 0.829107778372377, + "pass@128": 0.8502994011976048 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 727325, + "answer_accuracy": 0.9078489475155775, + "avg_response_len": 310.94691519212336, + "resp_tokens_sum": 249115743, + "pass_at_k": { + "pass@1": 0.9078489475155775, + "pass@2": 0.9314184621389209, + "pass@4": 0.9479047528888704, + "pass@8": 0.959567070014469, + "pass@16": 0.9676303174059113, + "pass@32": 0.9731903126693562, + "pass@64": 0.9775192818766577, + "pass@128": 0.9816264579006231 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9642024253731343, + "pass@2": 0.9741138059701494, + "pass@4": 0.9819364163434796, + "pass@8": 0.9881400872944226, + "pass@16": 0.9925379015562921, + "pass@32": 0.9955510391342886, + "pass@64": 0.997754923360079, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9912913602941177, + "pass@2": 0.9954405974988421, + "pass@4": 0.9979701380158362, + "pass@8": 0.9993477897991202, + "pass@16": 0.9999126470598457, + "pass@32": 0.999998726614258, + "pass@64": 0.999999999987784, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9933742088607594, + "pass@2": 0.9947567558307585, + "pass@4": 0.9957734057965857, + "pass@8": 0.9969989697228501, + "pass@16": 0.9982483505948174, + "pass@32": 0.9992648536542246, + "pass@64": 0.9999088007454459, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931493993993994, + "pass@2": 0.9975064877869996, + "pass@4": 0.999478633301468, + "pass@8": 0.9999675398986609, + "pass@16": 0.9999998926590724, + "pass@32": 0.9999999999998089, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930667202572347, + "pass@2": 0.9967814011190724, + "pass@4": 0.9985162387257863, + "pass@8": 0.9994198002264989, + "pass@16": 0.9999085821695214, + "pass@32": 0.999998609246854, + "pass@64": 0.9999999999866449, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.963640873015873, + "pass@2": 0.9752730127484065, + "pass@4": 0.9833094119187484, + "pass@8": 0.9885337281484384, + "pass@16": 0.9929572101573484, + "pass@32": 0.9967672908887808, + "pass@64": 0.9991148007644227, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9675793195718655, + "pass@2": 0.9831443087962632, + "pass@4": 0.9927234121537561, + "pass@8": 0.9962327927192676, + "pass@16": 0.9969067136543522, + "pass@32": 0.9969418396245668, + "pass@64": 0.9969418960244639, + "pass@128": 0.9969418960244648 + }, + "13": { + "pass@1": 0.953125, + "pass@2": 0.9724694243948675, + "pass@4": 0.9839528441467964, + "pass@8": 0.9914051147315478, + "pass@16": 0.9962751879157801, + "pass@32": 0.9989514073218897, + "pass@64": 0.9999069142994649, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9672663204747775, + "pass@2": 0.9819023224841703, + "pass@4": 0.9891655304029129, + "pass@8": 0.9926525053330164, + "pass@16": 0.9941520139636356, + "pass@32": 0.9947959873034944, + "pass@64": 0.9955489586054439, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9480372299382716, + "pass@2": 0.9670028768105379, + "pass@4": 0.9780621108704005, + "pass@8": 0.9842947803211456, + "pass@16": 0.9879716058879758, + "pass@32": 0.9901270313479683, + "pass@64": 0.9919070327284573, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.882973820058997, + "pass@2": 0.9149514842171274, + "pass@4": 0.9361787777080962, + "pass@8": 0.9505317245196182, + "pass@16": 0.95963740909398, + "pass@32": 0.9664879570651409, + "pass@64": 0.9730817511351202, + "pass@128": 0.9793510324483776 + }, + "17": { + "pass@1": 0.8233951430722891, + "pass@2": 0.8709084616023144, + "pass@4": 0.900955382553837, + "pass@8": 0.920503593025735, + "pass@16": 0.9343316545762004, + "pass@32": 0.9442573518899536, + "pass@64": 0.9519856298182777, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.7824344758064516, + "pass@2": 0.8496388430276858, + "pass@4": 0.8933080521184847, + "pass@8": 0.9212389467048724, + "pass@16": 0.9380265394631339, + "pass@32": 0.9476456964449388, + "pass@64": 0.9534177140834851, + "pass@128": 0.9580645161290322 + }, + "19": { + "pass@1": 0.5686367069486404, + "pass@2": 0.6704094012417636, + "pass@4": 0.7465739156094909, + "pass@8": 0.8053303353516325, + "pass@16": 0.8499870368405819, + "pass@32": 0.8823623062494177, + "pass@64": 0.907953968913336, + "pass@128": 0.9305135951661632 + }, + "20": { + "pass@1": 0.4545561186186186, + "pass@2": 0.5521487284528623, + "pass@4": 0.6327929909662195, + "pass@8": 0.6981138753732286, + "pass@16": 0.7452411820752879, + "pass@32": 0.7785809400350618, + "pass@64": 0.8062433242479462, + "pass@128": 0.8348348348348348 + } + } + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl b/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..55f4d9f2b234123780fad018de1577434a52b948 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0856a41fc6d7287904e654c1ed69815ee5a8f798e27f64f141ecc468cdecafbb +size 2712047402 diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..de17e44e3ca779bfc1a13df8e1f4b0b53bcf4e57 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9a1aa48fa3d36bc185d3cb2f6d68fc7570f68245d5df5e58c01a6513a1f052 +size 373191713 diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..decc2e6b95fe91dcfedff8a8c29972683cc27296 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5be0f75384b390f6e7f5e8b423d9aac35197e5a6a4420a6907c1ef23577736c4 +size 423962866 diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/composition_score_report.jsonl b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8ddb2d854a7cbb9e3fec46457bd5f77460f953aa --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition", "results_dir": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024126, "value_mismatch": 8591, "dependency_mismatch": 126456, "answer_mismatch": 7805, "extra_nodes": 12243, "missing_nodes": 9153, "perfect_with_extra": 1350, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337616, "dependency_mismatch": 40996, "extra_nodes": 3880, "perfect_with_extra": 438, "value_mismatch": 2510, "answer_mismatch": 2232, "missing_nodes": 2903}, "teachers_in_school": {"total": 390656, "perfect": 347014, "dependency_mismatch": 43146, "extra_nodes": 4142, "perfect_with_extra": 377, "value_mismatch": 2961, "answer_mismatch": 2821, "missing_nodes": 3349}, "movie_festival_awards": {"total": 382336, "perfect": 339496, "dependency_mismatch": 42314, "value_mismatch": 3120, "answer_mismatch": 2752, "extra_nodes": 4221, "missing_nodes": 2901, "perfect_with_extra": 535}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 619464, "value_mismatch": 264879, "dependency_mismatch": 640281, "answer_mismatch": 209944, "extra_nodes": 42043, "missing_nodes": 154289, "perfect_with_extra": 301, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 208964, "value_mismatch": 88980, "dependency_mismatch": 214494, "answer_mismatch": 71669, "extra_nodes": 15885, "missing_nodes": 55017, "perfect_with_extra": 70}, "movie_festival_awards": {"total": 418816, "perfect": 201602, "value_mismatch": 90032, "dependency_mismatch": 210582, "answer_mismatch": 71075, "missing_nodes": 48991, "extra_nodes": 13389, "perfect_with_extra": 47}, "teachers_in_school": {"total": 430464, "perfect": 208898, "extra_nodes": 12769, "perfect_with_extra": 184, "dependency_mismatch": 215205, "missing_nodes": 50281, "value_mismatch": 85867, "answer_mismatch": 67200}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1643590, "value_mismatch": 273470, "dependency_mismatch": 766737, "answer_mismatch": 217749, "extra_nodes": 54286, "missing_nodes": 163442, "perfect_with_extra": 1651, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 546580, "dependency_mismatch": 255490, "extra_nodes": 19765, "perfect_with_extra": 508, "value_mismatch": 91490, "answer_mismatch": 73901, "missing_nodes": 57920}, "teachers_in_school": {"total": 821120, "perfect": 555912, "dependency_mismatch": 258351, "extra_nodes": 16911, "perfect_with_extra": 561, "value_mismatch": 88828, "answer_mismatch": 70021, "missing_nodes": 53630}, "movie_festival_awards": {"total": 801152, "perfect": 541098, "dependency_mismatch": 252896, "value_mismatch": 93152, "answer_mismatch": 73827, "extra_nodes": 17610, "missing_nodes": 51892, "perfect_with_extra": 582}}}} diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/results.json b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..f487315907c0cd5ecaabd0b3badc0962fed2293b --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9619047619047619, + "pass@2": 0.9809523809523809, + "pass@4": 0.9968253968253968, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9580838323353293, + "pass@2": 0.9700598802395209, + "pass@4": 0.9850299401197605, + "pass@8": 0.9910179640718563, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8875739644970414, + "pass@2": 0.9556213017751479, + "pass@4": 0.9733727810650887, + "pass@8": 0.9970414201183432, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8076923076923077, + "pass@2": 0.8782051282051282, + "pass@4": 0.9358974358974359, + "pass@8": 0.9775641025641025, + "pass@16": 0.9871794871794872, + "pass@32": 0.9967948717948718, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7098591549295775, + "pass@2": 0.7887323943661971, + "pass@4": 0.8901408450704226, + "pass@8": 0.9323943661971831, + "pass@16": 0.9774647887323944, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.6430678466076696, + "pass@2": 0.696165191740413, + "pass@4": 0.7905604719764012, + "pass@8": 0.8790560471976401, + "pass@16": 0.9351032448377581, + "pass@32": 0.9528023598820059, + "pass@64": 0.9587020648967551, + "pass@128": 0.9646017699115044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9619047619047619, + "6": 0.9580838323353293, + "7": 0.8875739644970414, + "8": 0.8076923076923077, + "9": 0.7098591549295775, + "10": 0.6430678466076696 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9809523809523809, + "6": 0.9700598802395209, + "7": 0.9556213017751479, + "8": 0.8782051282051282, + "9": 0.7887323943661971, + "10": 0.696165191740413 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9850299401197605, + "7": 0.9733727810650887, + "8": 0.9358974358974359, + "9": 0.8901408450704226, + "10": 0.7905604719764012 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9910179640718563, + "7": 0.9970414201183432, + "8": 0.9775641025641025, + "9": 0.9323943661971831, + "10": 0.8790560471976401 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9970059880239521, + "7": 1.0, + "8": 0.9871794871794872, + "9": 0.9774647887323944, + "10": 0.9351032448377581 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9528023598820059 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9587020648967551 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9646017699115044 + } + }, + "total_pass_at_k": { + "pass@1": 0.8821344140493077, + "pass@2": 0.9165822357311719, + "pass@4": 0.9510300574130361, + "pass@8": 0.9739952718676123, + "pass@16": 0.9878419452887538, + "pass@32": 0.9935832489023978, + "pass@64": 0.994596420128335, + "pass@128": 0.9952718676122931 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9912280701754386, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9470588235294117, + "pass@2": 0.9823529411764705, + "pass@4": 0.9970588235294118, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9716088328075709, + "pass@2": 0.9779179810725552, + "pass@4": 0.9905362776025236, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.879746835443038, + "pass@2": 0.930379746835443, + "pass@4": 0.9683544303797469, + "pass@8": 0.9841772151898734, + "pass@16": 0.9936708860759493, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8018018018018018, + "pass@2": 0.8498498498498499, + "pass@4": 0.918918918918919, + "pass@8": 0.9669669669669669, + "pass@16": 0.990990990990991, + "pass@32": 0.996996996996997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7620578778135049, + "pass@2": 0.8231511254019293, + "pass@4": 0.9067524115755627, + "pass@8": 0.9678456591639871, + "pass@16": 0.9807073954983923, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6, + "pass@2": 0.6597014925373135, + "pass@4": 0.7492537313432835, + "pass@8": 0.8597014925373134, + "pass@16": 0.9194029850746268, + "pass@32": 0.9432835820895522, + "pass@64": 0.9641791044776119, + "pass@128": 0.9731343283582089 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9470588235294117, + "6": 0.9716088328075709, + "7": 0.879746835443038, + "8": 0.8018018018018018, + "9": 0.7620578778135049, + "10": 0.6 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9823529411764705, + "6": 0.9779179810725552, + "7": 0.930379746835443, + "8": 0.8498498498498499, + "9": 0.8231511254019293, + "10": 0.6597014925373135 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9905362776025236, + "7": 0.9683544303797469, + "8": 0.918918918918919, + "9": 0.9067524115755627, + "10": 0.7492537313432835 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9841772151898734, + "8": 0.9669669669669669, + "9": 0.9678456591639871, + "10": 0.8597014925373134 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.990990990990991, + "9": 0.9807073954983923, + "10": 0.9194029850746268 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9432835820895522 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 1.0, + "10": 0.9641791044776119 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9731343283582089 + } + }, + "total_pass_at_k": { + "pass@1": 0.8855038500167393, + "pass@2": 0.9139604954804151, + "pass@4": 0.9484432541011047, + "pass@8": 0.9755607633076665, + "pass@16": 0.9872782055574155, + "pass@32": 0.9926347505858721, + "pass@64": 0.995647807164379, + "pass@128": 0.9969869434214932 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9881656804733728, + "pass@4": 0.9940828402366864, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9681159420289855, + "pass@2": 0.9884057971014493, + "pass@4": 0.991304347826087, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9742120343839542, + "pass@2": 0.9799426934097422, + "pass@4": 0.9885386819484241, + "pass@8": 0.9914040114613181, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8583815028901735, + "pass@2": 0.9132947976878613, + "pass@4": 0.9508670520231214, + "pass@8": 0.9884393063583815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7971830985915493, + "pass@2": 0.8732394366197183, + "pass@4": 0.9577464788732394, + "pass@8": 0.9887323943661972, + "pass@16": 0.9971830985915493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7425149700598802, + "pass@2": 0.8053892215568862, + "pass@4": 0.8922155688622755, + "pass@8": 0.9730538922155688, + "pass@16": 0.9970059880239521, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.656441717791411, + "pass@2": 0.7208588957055214, + "pass@4": 0.8006134969325154, + "pass@8": 0.8650306748466258, + "pass@16": 0.9079754601226994, + "pass@32": 0.9447852760736196, + "pass@64": 0.9478527607361963, + "pass@128": 0.950920245398773 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9681159420289855, + "6": 0.9742120343839542, + "7": 0.8583815028901735, + "8": 0.7971830985915493, + "9": 0.7425149700598802, + "10": 0.656441717791411 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9884057971014493, + "6": 0.9799426934097422, + "7": 0.9132947976878613, + "8": 0.8732394366197183, + "9": 0.8053892215568862, + "10": 0.7208588957055214 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.991304347826087, + "6": 0.9885386819484241, + "7": 0.9508670520231214, + "8": 0.9577464788732394, + "9": 0.8922155688622755, + "10": 0.8006134969325154 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.9884393063583815, + "8": 0.9887323943661972, + "9": 0.9730538922155688, + "10": 0.8650306748466258 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9970059880239521, + "10": 0.9079754601226994 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9447852760736196 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9478527607361963 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.950920245398773 + } + }, + "total_pass_at_k": { + "pass@1": 0.8866317169069463, + "pass@2": 0.919397116644823, + "pass@4": 0.953473132372215, + "pass@8": 0.97870249017038, + "pass@16": 0.9888597640891219, + "pass@32": 0.9934469200524246, + "pass@64": 0.994429882044561, + "pass@128": 0.9947575360419397 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.989, + "pass@2": 0.993, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.959, + "pass@2": 0.984, + "pass@4": 0.995, + "pass@8": 0.998, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.968, + "pass@2": 0.976, + "pass@4": 0.988, + "pass@8": 0.994, + "pass@16": 0.998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.875, + "pass@2": 0.933, + "pass@4": 0.964, + "pass@8": 0.99, + "pass@16": 0.998, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.802, + "pass@2": 0.867, + "pass@4": 0.938, + "pass@8": 0.978, + "pass@16": 0.992, + "pass@32": 0.998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.737, + "pass@2": 0.805, + "pass@4": 0.896, + "pass@8": 0.957, + "pass@16": 0.985, + "pass@32": 0.997, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.633, + "pass@2": 0.692, + "pass@4": 0.78, + "pass@8": 0.868, + "pass@16": 0.921, + "pass@32": 0.947, + "pass@64": 0.957, + "pass@128": 0.963 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.989, + "5": 0.959, + "6": 0.968, + "7": 0.875, + "8": 0.802, + "9": 0.737, + "10": 0.633 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.993, + "5": 0.984, + "6": 0.976, + "7": 0.933, + "8": 0.867, + "9": 0.805, + "10": 0.692 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.995, + "6": 0.988, + "7": 0.964, + "8": 0.938, + "9": 0.896, + "10": 0.78 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.994, + "7": 0.99, + "8": 0.978, + "9": 0.957, + "10": 0.868 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.998, + "7": 0.998, + "8": 0.992, + "9": 0.985, + "10": 0.921 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.999, + "8": 0.998, + "9": 0.997, + "10": 0.947 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 0.999, + "10": 0.957 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.999, + "10": 0.963 + } + }, + "total_pass_at_k": { + "pass@1": 0.8847777777777778, + "pass@2": 0.9166666666666666, + "pass@4": 0.951, + "pass@8": 0.9761111111111112, + "pass@16": 0.988, + "pass@32": 0.9932222222222222, + "pass@64": 0.9948888888888889, + "pass@128": 0.9956666666666667 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6005665722379604, + "pass@2": 0.6742209631728046, + "pass@4": 0.773371104815864, + "pass@8": 0.8640226628895185, + "pass@16": 0.9178470254957507, + "pass@32": 0.9518413597733711, + "pass@64": 0.9745042492917847, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.6638418079096046, + "pass@2": 0.7231638418079096, + "pass@4": 0.8022598870056498, + "pass@8": 0.8813559322033898, + "pass@16": 0.9293785310734464, + "pass@32": 0.9491525423728814, + "pass@64": 0.963276836158192, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.5893416927899686, + "pass@2": 0.6394984326018809, + "pass@4": 0.7272727272727273, + "pass@8": 0.8056426332288401, + "pass@16": 0.8714733542319749, + "pass@32": 0.9122257053291536, + "pass@64": 0.9467084639498433, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.5607476635514018, + "pass@2": 0.5919003115264797, + "pass@4": 0.6417445482866043, + "pass@8": 0.7227414330218068, + "pass@16": 0.7757009345794392, + "pass@32": 0.8286604361370716, + "pass@64": 0.8847352024922118, + "pass@128": 0.9283489096573209 + }, + "15": { + "pass@1": 0.5057803468208093, + "pass@2": 0.5549132947976878, + "pass@4": 0.5953757225433526, + "pass@8": 0.6647398843930635, + "pass@16": 0.7456647398843931, + "pass@32": 0.8179190751445087, + "pass@64": 0.9104046242774566, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.4935483870967742, + "pass@2": 0.5193548387096775, + "pass@4": 0.5548387096774193, + "pass@8": 0.5870967741935483, + "pass@16": 0.6548387096774193, + "pass@32": 0.7032258064516129, + "pass@64": 0.7354838709677419, + "pass@128": 0.7709677419354839 + }, + "17": { + "pass@1": 0.5289017341040463, + "pass@2": 0.546242774566474, + "pass@4": 0.5809248554913294, + "pass@8": 0.6184971098265896, + "pass@16": 0.6705202312138728, + "pass@32": 0.7167630057803468, + "pass@64": 0.7803468208092486, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.4, + "pass@2": 0.4142857142857143, + "pass@4": 0.4514285714285714, + "pass@8": 0.49714285714285716, + "pass@16": 0.5457142857142857, + "pass@32": 0.6057142857142858, + "pass@64": 0.6714285714285714, + "pass@128": 0.7314285714285714 + }, + "19": { + "pass@1": 0.3063063063063063, + "pass@2": 0.33633633633633636, + "pass@4": 0.39039039039039036, + "pass@8": 0.4174174174174174, + "pass@16": 0.45645645645645644, + "pass@32": 0.5015015015015015, + "pass@64": 0.5555555555555556, + "pass@128": 0.5825825825825826 + }, + "20": { + "pass@1": 0.21621621621621623, + "pass@2": 0.24924924924924924, + "pass@4": 0.2822822822822823, + "pass@8": 0.3153153153153153, + "pass@16": 0.34234234234234234, + "pass@32": 0.39039039039039036, + "pass@64": 0.43843843843843844, + "pass@128": 0.4924924924924925 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6005665722379604, + "12": 0.6638418079096046, + "13": 0.5893416927899686, + "14": 0.5607476635514018, + "15": 0.5057803468208093, + "16": 0.4935483870967742, + "17": 0.5289017341040463, + "18": 0.4, + "19": 0.3063063063063063, + "20": 0.21621621621621623 + }, + "pass@2": { + "11": 0.6742209631728046, + "12": 0.7231638418079096, + "13": 0.6394984326018809, + "14": 0.5919003115264797, + "15": 0.5549132947976878, + "16": 0.5193548387096775, + "17": 0.546242774566474, + "18": 0.4142857142857143, + "19": 0.33633633633633636, + "20": 0.24924924924924924 + }, + "pass@4": { + "11": 0.773371104815864, + "12": 0.8022598870056498, + "13": 0.7272727272727273, + "14": 0.6417445482866043, + "15": 0.5953757225433526, + "16": 0.5548387096774193, + "17": 0.5809248554913294, + "18": 0.4514285714285714, + "19": 0.39039039039039036, + "20": 0.2822822822822823 + }, + "pass@8": { + "11": 0.8640226628895185, + "12": 0.8813559322033898, + "13": 0.8056426332288401, + "14": 0.7227414330218068, + "15": 0.6647398843930635, + "16": 0.5870967741935483, + "17": 0.6184971098265896, + "18": 0.49714285714285716, + "19": 0.4174174174174174, + "20": 0.3153153153153153 + }, + "pass@16": { + "11": 0.9178470254957507, + "12": 0.9293785310734464, + "13": 0.8714733542319749, + "14": 0.7757009345794392, + "15": 0.7456647398843931, + "16": 0.6548387096774193, + "17": 0.6705202312138728, + "18": 0.5457142857142857, + "19": 0.45645645645645644, + "20": 0.34234234234234234 + }, + "pass@32": { + "11": 0.9518413597733711, + "12": 0.9491525423728814, + "13": 0.9122257053291536, + "14": 0.8286604361370716, + "15": 0.8179190751445087, + "16": 0.7032258064516129, + "17": 0.7167630057803468, + "18": 0.6057142857142858, + "19": 0.5015015015015015, + "20": 0.39039039039039036 + }, + "pass@64": { + "11": 0.9745042492917847, + "12": 0.963276836158192, + "13": 0.9467084639498433, + "14": 0.8847352024922118, + "15": 0.9104046242774566, + "16": 0.7354838709677419, + "17": 0.7803468208092486, + "18": 0.6714285714285714, + "19": 0.5555555555555556, + "20": 0.43843843843843844 + }, + "pass@128": { + "11": 0.9830028328611898, + "12": 0.9745762711864406, + "13": 0.9717868338557993, + "14": 0.9283489096573209, + "15": 0.9450867052023122, + "16": 0.7709677419354839, + "17": 0.8294797687861272, + "18": 0.7314285714285714, + "19": 0.5825825825825826, + "20": 0.4924924924924925 + } + }, + "total_pass_at_k": { + "pass@1": 0.487369985141159, + "pass@2": 0.5260029717682021, + "pass@4": 0.5812778603268945, + "pass@8": 0.6389301634472511, + "pass@16": 0.6924219910846954, + "pass@32": 0.7390787518573552, + "pass@64": 0.787518573551263, + "pass@128": 0.8222882615156017 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6095238095238096, + "pass@2": 0.6952380952380952, + "pass@4": 0.8095238095238095, + "pass@8": 0.873015873015873, + "pass@16": 0.9333333333333333, + "pass@32": 0.9619047619047619, + "pass@64": 0.9841269841269841, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.617737003058104, + "pass@2": 0.6788990825688074, + "pass@4": 0.7675840978593272, + "pass@8": 0.8837920489296636, + "pass@16": 0.9327217125382263, + "pass@32": 0.9724770642201835, + "pass@64": 0.981651376146789, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.5864197530864198, + "pass@2": 0.6851851851851852, + "pass@4": 0.7623456790123457, + "pass@8": 0.8302469135802469, + "pass@16": 0.8796296296296297, + "pass@32": 0.9197530864197531, + "pass@64": 0.9506172839506173, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.56973293768546, + "pass@2": 0.599406528189911, + "pass@4": 0.6528189910979229, + "pass@8": 0.7151335311572701, + "pass@16": 0.7833827893175074, + "pass@32": 0.8486646884272997, + "pass@64": 0.9287833827893175, + "pass@128": 0.9584569732937686 + }, + "15": { + "pass@1": 0.5, + "pass@2": 0.5308641975308642, + "pass@4": 0.5802469135802469, + "pass@8": 0.6481481481481481, + "pass@16": 0.7314814814814815, + "pass@32": 0.7962962962962963, + "pass@64": 0.8765432098765432, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.5398230088495575, + "pass@2": 0.5781710914454278, + "pass@4": 0.6106194690265486, + "pass@8": 0.6460176991150443, + "pass@16": 0.7050147492625368, + "pass@32": 0.7463126843657817, + "pass@64": 0.7935103244837758, + "pass@128": 0.8230088495575221 + }, + "17": { + "pass@1": 0.4759036144578313, + "pass@2": 0.5090361445783133, + "pass@4": 0.536144578313253, + "pass@8": 0.5662650602409639, + "pass@16": 0.6234939759036144, + "pass@32": 0.7018072289156626, + "pass@64": 0.7680722891566265, + "pass@128": 0.7891566265060241 + }, + "18": { + "pass@1": 0.4290322580645161, + "pass@2": 0.45483870967741935, + "pass@4": 0.4870967741935484, + "pass@8": 0.535483870967742, + "pass@16": 0.5709677419354838, + "pass@32": 0.6387096774193548, + "pass@64": 0.7258064516129032, + "pass@128": 0.7741935483870968 + }, + "19": { + "pass@1": 0.2990936555891239, + "pass@2": 0.3564954682779456, + "pass@4": 0.3867069486404834, + "pass@8": 0.4199395770392749, + "pass@16": 0.459214501510574, + "pass@32": 0.4743202416918429, + "pass@64": 0.525679758308157, + "pass@128": 0.5770392749244713 + }, + "20": { + "pass@1": 0.1921921921921922, + "pass@2": 0.24324324324324326, + "pass@4": 0.2822822822822823, + "pass@8": 0.30930930930930933, + "pass@16": 0.3333333333333333, + "pass@32": 0.36036036036036034, + "pass@64": 0.4144144144144144, + "pass@128": 0.46846846846846846 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6095238095238096, + "12": 0.617737003058104, + "13": 0.5864197530864198, + "14": 0.56973293768546, + "15": 0.5, + "16": 0.5398230088495575, + "17": 0.4759036144578313, + "18": 0.4290322580645161, + "19": 0.2990936555891239, + "20": 0.1921921921921922 + }, + "pass@2": { + "11": 0.6952380952380952, + "12": 0.6788990825688074, + "13": 0.6851851851851852, + "14": 0.599406528189911, + "15": 0.5308641975308642, + "16": 0.5781710914454278, + "17": 0.5090361445783133, + "18": 0.45483870967741935, + "19": 0.3564954682779456, + "20": 0.24324324324324326 + }, + "pass@4": { + "11": 0.8095238095238095, + "12": 0.7675840978593272, + "13": 0.7623456790123457, + "14": 0.6528189910979229, + "15": 0.5802469135802469, + "16": 0.6106194690265486, + "17": 0.536144578313253, + "18": 0.4870967741935484, + "19": 0.3867069486404834, + "20": 0.2822822822822823 + }, + "pass@8": { + "11": 0.873015873015873, + "12": 0.8837920489296636, + "13": 0.8302469135802469, + "14": 0.7151335311572701, + "15": 0.6481481481481481, + "16": 0.6460176991150443, + "17": 0.5662650602409639, + "18": 0.535483870967742, + "19": 0.4199395770392749, + "20": 0.30930930930930933 + }, + "pass@16": { + "11": 0.9333333333333333, + "12": 0.9327217125382263, + "13": 0.8796296296296297, + "14": 0.7833827893175074, + "15": 0.7314814814814815, + "16": 0.7050147492625368, + "17": 0.6234939759036144, + "18": 0.5709677419354838, + "19": 0.459214501510574, + "20": 0.3333333333333333 + }, + "pass@32": { + "11": 0.9619047619047619, + "12": 0.9724770642201835, + "13": 0.9197530864197531, + "14": 0.8486646884272997, + "15": 0.7962962962962963, + "16": 0.7463126843657817, + "17": 0.7018072289156626, + "18": 0.6387096774193548, + "19": 0.4743202416918429, + "20": 0.36036036036036034 + }, + "pass@64": { + "11": 0.9841269841269841, + "12": 0.981651376146789, + "13": 0.9506172839506173, + "14": 0.9287833827893175, + "15": 0.8765432098765432, + "16": 0.7935103244837758, + "17": 0.7680722891566265, + "18": 0.7258064516129032, + "19": 0.525679758308157, + "20": 0.4144144144144144 + }, + "pass@128": { + "11": 0.9904761904761905, + "12": 0.9938837920489296, + "13": 0.9629629629629629, + "14": 0.9584569732937686, + "15": 0.9320987654320988, + "16": 0.8230088495575221, + "17": 0.7891566265060241, + "18": 0.7741935483870968, + "19": 0.5770392749244713, + "20": 0.46846846846846846 + } + }, + "total_pass_at_k": { + "pass@1": 0.48135696821515894, + "pass@2": 0.53239608801956, + "pass@4": 0.5864914425427873, + "pass@8": 0.6415036674816625, + "pass@16": 0.6940709046454768, + "pass@32": 0.7408312958435208, + "pass@64": 0.7937041564792175, + "pass@128": 0.8257946210268948 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6686746987951807, + "pass@2": 0.7228915662650602, + "pass@4": 0.8042168674698795, + "pass@8": 0.8825301204819277, + "pass@16": 0.9216867469879518, + "pass@32": 0.9578313253012049, + "pass@64": 0.9819277108433735, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.5987460815047022, + "pass@2": 0.658307210031348, + "pass@4": 0.7711598746081505, + "pass@8": 0.8369905956112853, + "pass@16": 0.9122257053291536, + "pass@32": 0.9467084639498433, + "pass@64": 0.9717868338557993, + "pass@128": 0.9811912225705329 + }, + "13": { + "pass@1": 0.6162464985994398, + "pass@2": 0.6750700280112045, + "pass@4": 0.7675070028011205, + "pass@8": 0.8487394957983193, + "pass@16": 0.9131652661064426, + "pass@32": 0.938375350140056, + "pass@64": 0.9635854341736695, + "pass@128": 0.9831932773109243 + }, + "14": { + "pass@1": 0.5672514619883041, + "pass@2": 0.6023391812865497, + "pass@4": 0.6374269005847953, + "pass@8": 0.716374269005848, + "pass@16": 0.7923976608187134, + "pass@32": 0.8713450292397661, + "pass@64": 0.9385964912280702, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.5545454545454546, + "pass@2": 0.5848484848484848, + "pass@4": 0.6242424242424243, + "pass@8": 0.6818181818181818, + "pass@16": 0.7545454545454545, + "pass@32": 0.8121212121212121, + "pass@64": 0.8727272727272727, + "pass@128": 0.9272727272727272 + }, + "16": { + "pass@1": 0.5156695156695157, + "pass@2": 0.5384615384615384, + "pass@4": 0.5726495726495726, + "pass@8": 0.6296296296296297, + "pass@16": 0.6837606837606838, + "pass@32": 0.7492877492877493, + "pass@64": 0.7977207977207977, + "pass@128": 0.8319088319088319 + }, + "17": { + "pass@1": 0.43478260869565216, + "pass@2": 0.45652173913043476, + "pass@4": 0.5186335403726708, + "pass@8": 0.5403726708074534, + "pass@16": 0.6180124223602484, + "pass@32": 0.6770186335403726, + "pass@64": 0.7298136645962733, + "pass@128": 0.7888198757763976 + }, + "18": { + "pass@1": 0.39705882352941174, + "pass@2": 0.4323529411764706, + "pass@4": 0.45294117647058824, + "pass@8": 0.5, + "pass@16": 0.5323529411764706, + "pass@32": 0.5823529411764706, + "pass@64": 0.6529411764705882, + "pass@128": 0.7205882352941176 + }, + "19": { + "pass@1": 0.2976190476190476, + "pass@2": 0.3601190476190476, + "pass@4": 0.39880952380952384, + "pass@8": 0.4375, + "pass@16": 0.4732142857142857, + "pass@32": 0.5148809523809523, + "pass@64": 0.5625, + "pass@128": 0.6101190476190477 + }, + "20": { + "pass@1": 0.19760479041916168, + "pass@2": 0.24251497005988024, + "pass@4": 0.2934131736526946, + "pass@8": 0.32934131736526945, + "pass@16": 0.3532934131736527, + "pass@32": 0.3772455089820359, + "pass@64": 0.4251497005988024, + "pass@128": 0.46407185628742514 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6686746987951807, + "12": 0.5987460815047022, + "13": 0.6162464985994398, + "14": 0.5672514619883041, + "15": 0.5545454545454546, + "16": 0.5156695156695157, + "17": 0.43478260869565216, + "18": 0.39705882352941174, + "19": 0.2976190476190476, + "20": 0.19760479041916168 + }, + "pass@2": { + "11": 0.7228915662650602, + "12": 0.658307210031348, + "13": 0.6750700280112045, + "14": 0.6023391812865497, + "15": 0.5848484848484848, + "16": 0.5384615384615384, + "17": 0.45652173913043476, + "18": 0.4323529411764706, + "19": 0.3601190476190476, + "20": 0.24251497005988024 + }, + "pass@4": { + "11": 0.8042168674698795, + "12": 0.7711598746081505, + "13": 0.7675070028011205, + "14": 0.6374269005847953, + "15": 0.6242424242424243, + "16": 0.5726495726495726, + "17": 0.5186335403726708, + "18": 0.45294117647058824, + "19": 0.39880952380952384, + "20": 0.2934131736526946 + }, + "pass@8": { + "11": 0.8825301204819277, + "12": 0.8369905956112853, + "13": 0.8487394957983193, + "14": 0.716374269005848, + "15": 0.6818181818181818, + "16": 0.6296296296296297, + "17": 0.5403726708074534, + "18": 0.5, + "19": 0.4375, + "20": 0.32934131736526945 + }, + "pass@16": { + "11": 0.9216867469879518, + "12": 0.9122257053291536, + "13": 0.9131652661064426, + "14": 0.7923976608187134, + "15": 0.7545454545454545, + "16": 0.6837606837606838, + "17": 0.6180124223602484, + "18": 0.5323529411764706, + "19": 0.4732142857142857, + "20": 0.3532934131736527 + }, + "pass@32": { + "11": 0.9578313253012049, + "12": 0.9467084639498433, + "13": 0.938375350140056, + "14": 0.8713450292397661, + "15": 0.8121212121212121, + "16": 0.7492877492877493, + "17": 0.6770186335403726, + "18": 0.5823529411764706, + "19": 0.5148809523809523, + "20": 0.3772455089820359 + }, + "pass@64": { + "11": 0.9819277108433735, + "12": 0.9717868338557993, + "13": 0.9635854341736695, + "14": 0.9385964912280702, + "15": 0.8727272727272727, + "16": 0.7977207977207977, + "17": 0.7298136645962733, + "18": 0.6529411764705882, + "19": 0.5625, + "20": 0.4251497005988024 + }, + "pass@128": { + "11": 0.9879518072289156, + "12": 0.9811912225705329, + "13": 0.9831932773109243, + "14": 0.9590643274853801, + "15": 0.9272727272727272, + "16": 0.8319088319088319, + "17": 0.7888198757763976, + "18": 0.7205882352941176, + "19": 0.6101190476190477, + "20": 0.46407185628742514 + } + }, + "total_pass_at_k": { + "pass@1": 0.4852809991079393, + "pass@2": 0.527802557240559, + "pass@4": 0.584299732381802, + "pass@8": 0.6407969075230449, + "pass@16": 0.695807314897413, + "pass@32": 0.7430865298840321, + "pass@64": 0.7900683913172762, + "pass@128": 0.8257508177222718 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.626, + "pass@2": 0.697, + "pass@4": 0.795, + "pass@8": 0.873, + "pass@16": 0.924, + "pass@32": 0.957, + "pass@64": 0.98, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.628, + "pass@2": 0.688, + "pass@4": 0.781, + "pass@8": 0.868, + "pass@16": 0.925, + "pass@32": 0.956, + "pass@64": 0.972, + "pass@128": 0.983 + }, + "13": { + "pass@1": 0.598, + "pass@2": 0.667, + "pass@4": 0.753, + "pass@8": 0.829, + "pass@16": 0.889, + "pass@32": 0.924, + "pass@64": 0.954, + "pass@128": 0.973 + }, + "14": { + "pass@1": 0.566, + "pass@2": 0.598, + "pass@4": 0.644, + "pass@8": 0.718, + "pass@16": 0.784, + "pass@32": 0.85, + "pass@64": 0.918, + "pass@128": 0.949 + }, + "15": { + "pass@1": 0.52, + "pass@2": 0.557, + "pass@4": 0.6, + "pass@8": 0.665, + "pass@16": 0.744, + "pass@32": 0.809, + "pass@64": 0.887, + "pass@128": 0.935 + }, + "16": { + "pass@1": 0.517, + "pass@2": 0.546, + "pass@4": 0.58, + "pass@8": 0.622, + "pass@16": 0.682, + "pass@32": 0.734, + "pass@64": 0.777, + "pass@128": 0.81 + }, + "17": { + "pass@1": 0.481, + "pass@2": 0.505, + "pass@4": 0.546, + "pass@8": 0.576, + "pass@16": 0.638, + "pass@32": 0.699, + "pass@64": 0.76, + "pass@128": 0.803 + }, + "18": { + "pass@1": 0.408, + "pass@2": 0.433, + "pass@4": 0.463, + "pass@8": 0.51, + "pass@16": 0.549, + "pass@32": 0.608, + "pass@64": 0.682, + "pass@128": 0.741 + }, + "19": { + "pass@1": 0.301, + "pass@2": 0.351, + "pass@4": 0.392, + "pass@8": 0.425, + "pass@16": 0.463, + "pass@32": 0.497, + "pass@64": 0.548, + "pass@128": 0.59 + }, + "20": { + "pass@1": 0.202, + "pass@2": 0.245, + "pass@4": 0.286, + "pass@8": 0.318, + "pass@16": 0.343, + "pass@32": 0.376, + "pass@64": 0.426, + "pass@128": 0.475 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.626, + "12": 0.628, + "13": 0.598, + "14": 0.566, + "15": 0.52, + "16": 0.517, + "17": 0.481, + "18": 0.408, + "19": 0.301, + "20": 0.202 + }, + "pass@2": { + "11": 0.697, + "12": 0.688, + "13": 0.667, + "14": 0.598, + "15": 0.557, + "16": 0.546, + "17": 0.505, + "18": 0.433, + "19": 0.351, + "20": 0.245 + }, + "pass@4": { + "11": 0.795, + "12": 0.781, + "13": 0.753, + "14": 0.644, + "15": 0.6, + "16": 0.58, + "17": 0.546, + "18": 0.463, + "19": 0.392, + "20": 0.286 + }, + "pass@8": { + "11": 0.873, + "12": 0.868, + "13": 0.829, + "14": 0.718, + "15": 0.665, + "16": 0.622, + "17": 0.576, + "18": 0.51, + "19": 0.425, + "20": 0.318 + }, + "pass@16": { + "11": 0.924, + "12": 0.925, + "13": 0.889, + "14": 0.784, + "15": 0.744, + "16": 0.682, + "17": 0.638, + "18": 0.549, + "19": 0.463, + "20": 0.343 + }, + "pass@32": { + "11": 0.957, + "12": 0.956, + "13": 0.924, + "14": 0.85, + "15": 0.809, + "16": 0.734, + "17": 0.699, + "18": 0.608, + "19": 0.497, + "20": 0.376 + }, + "pass@64": { + "11": 0.98, + "12": 0.972, + "13": 0.954, + "14": 0.918, + "15": 0.887, + "16": 0.777, + "17": 0.76, + "18": 0.682, + "19": 0.548, + "20": 0.426 + }, + "pass@128": { + "11": 0.987, + "12": 0.983, + "13": 0.973, + "14": 0.949, + "15": 0.935, + "16": 0.81, + "17": 0.803, + "18": 0.741, + "19": 0.59, + "20": 0.475 + } + }, + "total_pass_at_k": { + "pass@1": 0.4847, + "pass@2": 0.5287, + "pass@4": 0.584, + "pass@8": 0.6404, + "pass@16": 0.6941, + "pass@32": 0.741, + "pass@64": 0.7904, + "pass@128": 0.8246 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/summary.csv b/Qwen3-100M-midtrain-diff_ar-block16/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8d6d1a6aac473a2b8b345fcba7129fadc98a1e92 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-38646,0.9932248263888889,0.83598125,0.9104650493421053,0.0,0.0,0.0,200.3281736111111,378.03591796875,1152000,1280000,2432000 diff --git a/Qwen3-100M-midtrain-diff_ar-block16/composition/summary.json b/Qwen3-100M-midtrain-diff_ar-block16/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dbc8e8677e6eec93bcc4045fd1120cf180395348 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block16/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-38646", + "id_acc": 0.9932248263888889, + "ood_acc": 0.83598125, + "total_acc": 0.9104650493421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.3281736111111, + "ood_avg_resp_len": 378.03591796875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 735827, + "answer_accuracy": 0.9087335500316156, + "avg_response_len": 280.81283097533986, + "resp_tokens_sum": 227382012, + "pass_at_k": { + "pass@1": 0.9087335500316156, + "pass@2": 0.9306044600025811, + "pass@4": 0.9464375050603238, + "pass@8": 0.9578451948086137, + "pass@16": 0.9661519899120048, + "pass@32": 0.9723665816849368, + "pass@64": 0.9774353954902267, + "pass@128": 0.9818210559595321 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9643482669616519, + "pass@2": 0.9724997386941677, + "pass@4": 0.9787036025917114, + "pass@8": 0.9837184467724707, + "pass@16": 0.9876176547055514, + "pass@32": 0.990048743242238, + "pass@64": 0.9918928791392225, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998805428134556, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967757936507936, + "pass@2": 0.9988044463192103, + "pass@4": 0.9997930303354938, + "pass@8": 0.9999940728357846, + "pass@16": 0.999999996928095, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981521332335329, + "pass@2": 0.999078737210618, + "pass@4": 0.9997273029868273, + "pass@8": 0.9999777200738017, + "pass@16": 0.9999998973084382, + "pass@32": 0.999999999999679, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998959874260355, + "pass@2": 0.9998143607603783, + "pass@4": 0.9999912155026479, + "pass@8": 0.9999999836752375, + "pass@16": 0.9999999999999905, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960687099358975, + "pass@2": 0.9985512252675145, + "pass@4": 0.9996649123787412, + "pass@8": 0.9999730867256917, + "pass@16": 0.9999998586577866, + "pass@32": 0.9999999999993815, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9940801056338028, + "pass@2": 0.9950447765332151, + "pass@4": 0.9956578631896364, + "pass@8": 0.996375867480201, + "pass@16": 0.9969724240620078, + "pass@32": 0.9971729386969103, + "pass@64": 0.9971830959162251, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9711623583569405, + "pass@2": 0.9805191998840088, + "pass@4": 0.986303336286222, + "pass@8": 0.989850641477996, + "pass@16": 0.99240868699634, + "pass@32": 0.994293234140082, + "pass@64": 0.9956686613210934, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9658368644067796, + "pass@2": 0.979869405889942, + "pass@4": 0.9899045508612269, + "pass@8": 0.9949535165900368, + "pass@16": 0.9969359713595375, + "pass@32": 0.9982218320963844, + "pass@64": 0.9992952648675736, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9498187695924765, + "pass@2": 0.9693292893639082, + "pass@4": 0.9815207164543299, + "pass@8": 0.9885514952344922, + "pass@16": 0.993030758542598, + "pass@32": 0.9957866897580077, + "pass@64": 0.9968002718717178, + "pass@128": 0.9968652037617555 + }, + "14": { + "pass@1": 0.9244548286604362, + "pass@2": 0.9464950971373904, + "pass@4": 0.962845547343965, + "pass@8": 0.9744775229835171, + "pass@16": 0.9831241464032126, + "pass@32": 0.9892894037314296, + "pass@64": 0.9925585252418944, + "pass@128": 0.9937694704049844 + }, + "15": { + "pass@1": 0.9763367052023122, + "pass@2": 0.9890843327568163, + "pass@4": 0.995195050131017, + "pass@8": 0.9987062171803248, + "pass@16": 0.9998947926322176, + "pass@32": 0.9999995163740522, + "pass@64": 0.9999999999997273, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8670362903225807, + "pass@2": 0.9042973710947418, + "pass@4": 0.9324377792493678, + "pass@8": 0.9511663793058849, + "pass@16": 0.9632457765868611, + "pass@32": 0.972380137786845, + "pass@64": 0.9803184368887754, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.8796739523121387, + "pass@2": 0.9130583354194169, + "pass@4": 0.9346565702950422, + "pass@8": 0.949022076996675, + "pass@16": 0.9577801252538675, + "pass@32": 0.9642535207391926, + "pass@64": 0.9711417638827387, + "pass@128": 0.976878612716763 + }, + "18": { + "pass@1": 0.7535491071428572, + "pass@2": 0.8142751687289084, + "pass@4": 0.854199288124698, + "pass@8": 0.8827058936508151, + "pass@16": 0.9034225204407647, + "pass@32": 0.9183155907983432, + "pass@64": 0.9304605804851537, + "pass@128": 0.9428571428571428 + }, + "19": { + "pass@1": 0.5902073948948949, + "pass@2": 0.6823130955758907, + "pass@4": 0.7493216109810598, + "pass@8": 0.7968110404034479, + "pass@16": 0.8325350334716398, + "pass@32": 0.859961150550141, + "pass@64": 0.8827473329655091, + "pass@128": 0.9009009009009009 + }, + "20": { + "pass@1": 0.4485735735735736, + "pass@2": 0.5452467723629141, + "pass@4": 0.6280490980294128, + "pass@8": 0.6976146762314849, + "pass@16": 0.7542370813489142, + "pass@32": 0.799241057750153, + "pass@64": 0.8368248092112288, + "pass@128": 0.8708708708708709 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 751099, + "answer_accuracy": 0.9147250097427904, + "avg_response_len": 290.0505114964926, + "resp_tokens_sum": 238166276, + "pass_at_k": { + "pass@1": 0.9147250097427904, + "pass@2": 0.9368544058892373, + "pass@4": 0.9526741754464987, + "pass@8": 0.9638412119281327, + "pass@16": 0.9716894634319847, + "pass@32": 0.9774218920691676, + "pass@64": 0.9817877251624777, + "pass@128": 0.9851909586905689 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9476131134969326, + "pass@2": 0.9578519002705184, + "pass@4": 0.9660574474778076, + "pass@8": 0.9719991236070697, + "pass@16": 0.9763583249372905, + "pass@32": 0.980656765738378, + "pass@64": 0.9852951841811889, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9992185635886424, + "pass@4": 0.9998076788696866, + "pass@8": 0.9999893019044129, + "pass@16": 0.9999999789617312, + "pass@32": 0.9999999999999926, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9955842391304348, + "pass@2": 0.9970807657195024, + "pass@4": 0.9974464667460045, + "pass@8": 0.9977680565376104, + "pass@16": 0.9983124881128989, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994627507163324, + "pass@2": 0.9999027028856349, + "pass@4": 0.9999971459513118, + "pass@8": 0.9999999985260104, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984420158959537, + "pass@2": 0.9991658037868099, + "pass@4": 0.9997657894822396, + "pass@8": 0.999983097148883, + "pass@16": 0.9999999407613452, + "pass@32": 0.9999999999999076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9972711267605634, + "pass@2": 0.9988802401020296, + "pass@4": 0.9997421626874106, + "pass@8": 0.9999832931204404, + "pass@16": 0.9999999422581453, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965147829341318, + "pass@2": 0.9973931320430004, + "pass@4": 0.9978456220292824, + "pass@8": 0.998468082786387, + "pass@16": 0.9992530482905866, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9768213478915663, + "pass@2": 0.982426507209942, + "pass@4": 0.9875867705919291, + "pass@8": 0.9916699109297928, + "pass@16": 0.9941198985224184, + "pass@32": 0.9952745122110707, + "pass@64": 0.9962408632343975, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.965492750783699, + "pass@2": 0.9797927517093281, + "pass@4": 0.9877445484870819, + "pass@8": 0.9923678770900078, + "pass@16": 0.9955227193860775, + "pass@32": 0.9973880317416322, + "pass@64": 0.9984300583339356, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9623599439775911, + "pass@2": 0.9779697804318578, + "pass@4": 0.9872410336838148, + "pass@8": 0.9920194344744313, + "pass@16": 0.9942491977157041, + "pass@32": 0.9950794358980206, + "pass@64": 0.9957983155432268, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.9557291666666666, + "pass@2": 0.9763930618870013, + "pass@4": 0.9898799393496869, + "pass@8": 0.9971921128374414, + "pass@16": 0.9996866455629769, + "pass@32": 0.9999957420781709, + "pass@64": 0.9999999999150341, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9709043560606061, + "pass@2": 0.9843362264376043, + "pass@4": 0.9913319070911589, + "pass@8": 0.9957115636967746, + "pass@16": 0.9984277498659098, + "pass@32": 0.9997408383719552, + "pass@64": 0.9999955357422065, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9107015669515669, + "pass@2": 0.9377124133521775, + "pass@4": 0.9564160948898481, + "pass@8": 0.9692630161138946, + "pass@16": 0.9776624723599159, + "pass@32": 0.9832541934776856, + "pass@64": 0.9874615955124632, + "pass@128": 0.9914529914529915 + }, + "17": { + "pass@1": 0.8672360248447205, + "pass@2": 0.9112839750330124, + "pass@4": 0.9387636921937549, + "pass@8": 0.9558194693071742, + "pass@16": 0.9657225480978907, + "pass@32": 0.9716417279156446, + "pass@64": 0.9759516660247486, + "pass@128": 0.9782608695652174 + }, + "18": { + "pass@1": 0.7839613970588235, + "pass@2": 0.8523347035664656, + "pass@4": 0.8947464600013231, + "pass@8": 0.9214377697528552, + "pass@16": 0.9388117402083257, + "pass@32": 0.951005420422579, + "pass@64": 0.9607119551856902, + "pass@128": 0.9705882352941176 + }, + "19": { + "pass@1": 0.5933547247023809, + "pass@2": 0.6915281830005624, + "pass@4": 0.766301561356169, + "pass@8": 0.8222817559953494, + "pass@16": 0.8649021620408612, + "pass@32": 0.8975421565193381, + "pass@64": 0.9221817078073432, + "pass@128": 0.9404761904761905 + }, + "20": { + "pass@1": 0.4492889221556886, + "pass@2": 0.5484648321467304, + "pass@4": 0.6333525578202419, + "pass@8": 0.7015890174857582, + "pass@16": 0.7546736568184237, + "pass@32": 0.796908044139665, + "pass@64": 0.829107778372377, + "pass@128": 0.8502994011976048 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 727325, + "answer_accuracy": 0.9078489475155775, + "avg_response_len": 310.94691519212336, + "resp_tokens_sum": 249115743, + "pass_at_k": { + "pass@1": 0.9078489475155775, + "pass@2": 0.9314184621389209, + "pass@4": 0.9479047528888704, + "pass@8": 0.959567070014469, + "pass@16": 0.9676303174059113, + "pass@32": 0.9731903126693562, + "pass@64": 0.9775192818766577, + "pass@128": 0.9816264579006231 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9642024253731343, + "pass@2": 0.9741138059701494, + "pass@4": 0.9819364163434796, + "pass@8": 0.9881400872944226, + "pass@16": 0.9925379015562921, + "pass@32": 0.9955510391342886, + "pass@64": 0.997754923360079, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9912913602941177, + "pass@2": 0.9954405974988421, + "pass@4": 0.9979701380158362, + "pass@8": 0.9993477897991202, + "pass@16": 0.9999126470598457, + "pass@32": 0.999998726614258, + "pass@64": 0.999999999987784, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9933742088607594, + "pass@2": 0.9947567558307585, + "pass@4": 0.9957734057965857, + "pass@8": 0.9969989697228501, + "pass@16": 0.9982483505948174, + "pass@32": 0.9992648536542246, + "pass@64": 0.9999088007454459, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931493993993994, + "pass@2": 0.9975064877869996, + "pass@4": 0.999478633301468, + "pass@8": 0.9999675398986609, + "pass@16": 0.9999998926590724, + "pass@32": 0.9999999999998089, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930667202572347, + "pass@2": 0.9967814011190724, + "pass@4": 0.9985162387257863, + "pass@8": 0.9994198002264989, + "pass@16": 0.9999085821695214, + "pass@32": 0.999998609246854, + "pass@64": 0.9999999999866449, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.963640873015873, + "pass@2": 0.9752730127484065, + "pass@4": 0.9833094119187484, + "pass@8": 0.9885337281484384, + "pass@16": 0.9929572101573484, + "pass@32": 0.9967672908887808, + "pass@64": 0.9991148007644227, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9675793195718655, + "pass@2": 0.9831443087962632, + "pass@4": 0.9927234121537561, + "pass@8": 0.9962327927192676, + "pass@16": 0.9969067136543522, + "pass@32": 0.9969418396245668, + "pass@64": 0.9969418960244639, + "pass@128": 0.9969418960244648 + }, + "13": { + "pass@1": 0.953125, + "pass@2": 0.9724694243948675, + "pass@4": 0.9839528441467964, + "pass@8": 0.9914051147315478, + "pass@16": 0.9962751879157801, + "pass@32": 0.9989514073218897, + "pass@64": 0.9999069142994649, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9672663204747775, + "pass@2": 0.9819023224841703, + "pass@4": 0.9891655304029129, + "pass@8": 0.9926525053330164, + "pass@16": 0.9941520139636356, + "pass@32": 0.9947959873034944, + "pass@64": 0.9955489586054439, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9480372299382716, + "pass@2": 0.9670028768105379, + "pass@4": 0.9780621108704005, + "pass@8": 0.9842947803211456, + "pass@16": 0.9879716058879758, + "pass@32": 0.9901270313479683, + "pass@64": 0.9919070327284573, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.882973820058997, + "pass@2": 0.9149514842171274, + "pass@4": 0.9361787777080962, + "pass@8": 0.9505317245196182, + "pass@16": 0.95963740909398, + "pass@32": 0.9664879570651409, + "pass@64": 0.9730817511351202, + "pass@128": 0.9793510324483776 + }, + "17": { + "pass@1": 0.8233951430722891, + "pass@2": 0.8709084616023144, + "pass@4": 0.900955382553837, + "pass@8": 0.920503593025735, + "pass@16": 0.9343316545762004, + "pass@32": 0.9442573518899536, + "pass@64": 0.9519856298182777, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.7824344758064516, + "pass@2": 0.8496388430276858, + "pass@4": 0.8933080521184847, + "pass@8": 0.9212389467048724, + "pass@16": 0.9380265394631339, + "pass@32": 0.9476456964449388, + "pass@64": 0.9534177140834851, + "pass@128": 0.9580645161290322 + }, + "19": { + "pass@1": 0.5686367069486404, + "pass@2": 0.6704094012417636, + "pass@4": 0.7465739156094909, + "pass@8": 0.8053303353516325, + "pass@16": 0.8499870368405819, + "pass@32": 0.8823623062494177, + "pass@64": 0.907953968913336, + "pass@128": 0.9305135951661632 + }, + "20": { + "pass@1": 0.4545561186186186, + "pass@2": 0.5521487284528623, + "pass@4": 0.6327929909662195, + "pass@8": 0.6981138753732286, + "pass@16": 0.7452411820752879, + "pass@32": 0.7785809400350618, + "pass@64": 0.8062433242479462, + "pass@128": 0.8348348348348348 + } + } + } + } + } +] \ No newline at end of file diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl b/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9d524c9f7c0a0a7ae0315e73534bfed8e3fbfc3e --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad599636b8c33c4e7935cbaeff8d20f8a960498bbfaef9961b706eaa1c1595f1 +size 1789731521 diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_metrics.json b/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d24dbcb56ff4508f4e1d1fbff6b3382132618a43 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-38646", + "id": { + "answer_accuracy": 0.9930946180555555, + "avg_response_len": 200.28550694444445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9930946180555555, + "pass@2": 0.9949989473972002, + "pass@4": 0.9963422400845728, + "pass@8": 0.9972624951869838, + "pass@16": 0.9979487555732529, + "pass@32": 0.9985491774462739, + "pass@64": 0.9991352162559932, + "pass@128": 0.9996666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9540625, + "pass@2": 0.9639723179133853, + "pass@4": 0.9723782580614919, + "pass@8": 0.9787925265522301, + "pass@16": 0.9837687276335041, + "pass@32": 0.9881571311082412, + "pass@64": 0.9927600328966237, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99946875, + "pass@2": 0.9998220964566928, + "pass@4": 0.9999744520997375, + "pass@8": 0.9999994736310946, + "pass@16": 0.9999999998889773, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999890625, + "pass@2": 0.9999948326771654, + "pass@4": 0.9999999880952382, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9944765625, + "pass@2": 0.9970152559055121, + "pass@4": 0.9982060622422197, + "pass@8": 0.9988426718080923, + "pass@16": 0.9993716905884307, + "pass@32": 0.9998027430992852, + "pass@64": 0.9999860378787945, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999140625, + "pass@2": 0.9999985236220473, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99834375, + "pass@2": 0.9991849163385828, + "pass@4": 0.9996791599175103, + "pass@8": 0.9999399336710925, + "pass@16": 0.9999976797361912, + "pass@32": 0.9999999983745138, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9961796875, + "pass@2": 0.9982639025590552, + "pass@4": 0.9995023362392201, + "pass@8": 0.999940916408823, + "pass@16": 0.9999986572834463, + "pass@32": 0.9999999995728239, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.995515625, + "pass@2": 0.9967386811023622, + "pass@4": 0.9973399041994752, + "pass@8": 0.9978469346115266, + "pass@16": 0.9984020450287275, + "pass@32": 0.9989827248615993, + "pass@64": 0.9994708755285203, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9540625, + "2": 1.0, + "3": 0.99946875, + "4": 0.999890625, + "5": 0.9944765625, + "6": 0.9999140625, + "7": 0.99834375, + "8": 0.9961796875, + "9": 0.995515625 + }, + "per_op_avg_response_len": { + "10": 281.6685703125, + "2": 115.6646328125, + "3": 144.9749609375, + "4": 159.97440625, + "5": 173.51565625, + "6": 198.98446875, + "7": 224.252796875, + "8": 243.7776015625, + "9": 259.75646875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.6685703125, + "2": 115.6646328125, + "3": 144.9749609375, + "4": 159.97440625, + "5": 173.51565625, + "6": 198.98446875, + "7": 224.252796875, + "8": 243.7776015625, + "9": 259.75646875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376657, + "answer_accuracy": 0.9937969647078689, + "avg_response_len": 190.8121912993921, + "resp_tokens_sum": 72319347, + "pass_at_k": { + "pass@1": 0.9937969647078689, + "pass@2": 0.9952953437336292, + "pass@4": 0.9963559719042212, + "pass@8": 0.9972608428511113, + "pass@16": 0.9980648749134898, + "pass@32": 0.9986967786412437, + "pass@64": 0.9991892593315244, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954484697640118, + "pass@2": 0.9629707860079438, + "pass@4": 0.9709464638269774, + "pass@8": 0.9782191819839355, + "pass@16": 0.984593484638989, + "pass@32": 0.9892985145351701, + "pass@64": 0.9930035863666121, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9980902777777778, + "pass@2": 0.9997808867641542, + "pass@4": 0.9999950205629058, + "pass@8": 0.9999999974305525, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 0.9999977898533642, + "pass@4": 0.9999999997193464, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9991910133136095, + "pass@2": 0.9999253803056423, + "pass@4": 0.9999992966855474, + "pass@8": 0.9999999999707041, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9977964743589743, + "pass@2": 0.9990386192711489, + "pass@4": 0.9997245203123646, + "pass@8": 0.9999788364839716, + "pass@16": 0.9999999148409177, + "pass@32": 0.9999999999998113, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962367957746479, + "pass@2": 0.9972319646223802, + "pass@4": 0.9975970072402921, + "pass@8": 0.9979708642982112, + "pass@16": 0.9985716335094632, + "pass@32": 0.9993491975474896, + "pass@64": 0.9999188200066541, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387649, + "answer_accuracy": 0.9923026908584535, + "avg_response_len": 198.26654652686761, + "resp_tokens_sum": 77454016, + "pass_at_k": { + "pass@1": 0.9923026908584535, + "pass@2": 0.9942880249558829, + "pass@4": 0.9956853823809375, + "pass@8": 0.9965860027278677, + "pass@16": 0.997238113916091, + "pass@32": 0.9979263130447362, + "pass@64": 0.9987312934569264, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9433713573619632, + "pass@2": 0.9535782540698516, + "pass@4": 0.9618546829460732, + "pass@8": 0.968359742151891, + "pass@16": 0.9741511694787096, + "pass@32": 0.9805862212217205, + "pass@64": 0.9881224160445984, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983901515151515, + "pass@2": 0.999460898353615, + "pass@4": 0.9999225821204167, + "pass@8": 0.9999984049427112, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999813, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935914855072464, + "pass@2": 0.9972134257674314, + "pass@4": 0.9991986126734157, + "pass@8": 0.9998860811105759, + "pass@16": 0.9999969943734291, + "pass@32": 0.9999999992176973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999328438395415, + "pass@2": 0.99999894242267, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981484826589595, + "pass@2": 0.9991423353511447, + "pass@4": 0.9997656553768929, + "pass@8": 0.9999830971478822, + "pass@16": 0.9999999407613452, + "pass@32": 0.9999999999999076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9978213028169014, + "pass@2": 0.998696212709327, + "pass@4": 0.9994122448602376, + "pass@8": 0.9998848700630322, + "pass@16": 0.9999964254002791, + "pass@32": 0.9999999987973975, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9976609281437125, + "pass@2": 0.9988149930454053, + "pass@4": 0.9995776498184733, + "pass@8": 0.9999453661202221, + "pass@16": 0.9999993082440582, + "pass@32": 0.9999999999733661, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379739, + "answer_accuracy": 0.9932075451958486, + "avg_response_len": 211.73925813943757, + "resp_tokens_sum": 80955541, + "pass_at_k": { + "pass@1": 0.9932075451958486, + "pass@2": 0.9954315238078923, + "pass@4": 0.9969997793525728, + "pass@8": 0.9979553467275731, + "pass@16": 0.9985597528853449, + "pass@32": 0.9990392795604974, + "pass@64": 0.9994943561408621, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9640391791044776, + "pass@2": 0.9751006287460335, + "pass@4": 0.9840680005484389, + "pass@8": 0.989525217666148, + "pass@16": 0.9922932987785924, + "pass@32": 0.9943696315299635, + "pass@64": 0.9970265954859788, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998857821637427, + "pass@2": 0.9999978415527007, + "pass@4": 0.9999999997259115, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920266544117647, + "pass@2": 0.9942518961324689, + "pass@4": 0.9955414982134585, + "pass@8": 0.9967116901009395, + "pass@16": 0.998155080969405, + "pass@32": 0.9994198334387632, + "pass@64": 0.9999589349376307, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.9999988356640751, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976513053797469, + "pass@2": 0.9984395245689226, + "pass@4": 0.9992420280930073, + "pass@8": 0.9998284242652762, + "pass@16": 0.9999927222555876, + "pass@32": 0.9999999948561573, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9929147897897898, + "pass@2": 0.9970771706746117, + "pass@4": 0.9993902071362702, + "pass@8": 0.9999651367971397, + "pass@16": 0.9999998583663093, + "pass@32": 0.9999999999994196, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9923884646302251, + "pass@2": 0.9939457427146365, + "pass@4": 0.994643188391644, + "pass@8": 0.995451850487162, + "pass@16": 0.9964930745316808, + "pass@32": 0.9974718962737498, + "pass@64": 0.9983913004056533, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.870721875, + "avg_response_len": 379.828709375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.870721875, + "pass@2": 0.9070165846456469, + "pass@4": 0.9310222992875821, + "pass@8": 0.9473439413551321, + "pass@16": 0.9587791727410775, + "pass@32": 0.9673418489786182, + "pass@64": 0.9738290079881573, + "pass@128": 0.9788 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9725234375, + "pass@2": 0.9814163385826772, + "pass@4": 0.9876329908136482, + "pass@8": 0.9918884097903063, + "pass@16": 0.9949111139255706, + "pass@32": 0.9969289474459831, + "pass@64": 0.9981950900579236, + "pass@128": 0.999 + }, + "12": { + "pass@1": 0.9633828125, + "pass@2": 0.9799354084645666, + "pass@4": 0.9903667950881146, + "pass@8": 0.9952444450104463, + "pass@16": 0.9974553279156712, + "pass@32": 0.9988006564676271, + "pass@64": 0.9996719396956972, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9512578125, + "pass@2": 0.9713690944881891, + "pass@4": 0.9840455555868022, + "pass@8": 0.9908545724750295, + "pass@16": 0.9943492242524582, + "pass@32": 0.9960756104075091, + "pass@64": 0.9969796143219964, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.9520390625, + "pass@2": 0.9693324311023619, + "pass@4": 0.9814703537682793, + "pass@8": 0.9892758228409231, + "pass@16": 0.9936319146442153, + "pass@32": 0.9960982517605781, + "pass@64": 0.9977339665394206, + "pass@128": 0.999 + }, + "15": { + "pass@1": 0.959546875, + "pass@2": 0.9762183809055123, + "pass@4": 0.9848403532995874, + "pass@8": 0.9897748141534157, + "pass@16": 0.9926387666188844, + "pass@32": 0.9944456451139438, + "pass@64": 0.9959166661010433, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.896828125, + "pass@2": 0.9233396899606302, + "pass@4": 0.9414247777465318, + "pass@8": 0.953458790870688, + "pass@16": 0.9612533257789929, + "pass@32": 0.9673482732504611, + "pass@64": 0.9725713413565941, + "pass@128": 0.977 + }, + "17": { + "pass@1": 0.8799609375, + "pass@2": 0.9119715797244108, + "pass@4": 0.9323001194225726, + "pass@8": 0.9465483790840621, + "pass@16": 0.9567859939940381, + "pass@32": 0.9648555078508166, + "pass@64": 0.9711329473830459, + "pass@128": 0.976 + }, + "18": { + "pass@1": 0.83540625, + "pass@2": 0.8872344980314973, + "pass@4": 0.9175925875515554, + "pass@8": 0.9373478636623114, + "pass@16": 0.9512725593857325, + "pass@32": 0.9618185543165673, + "pass@64": 0.9696181077106221, + "pass@128": 0.975 + }, + "19": { + "pass@1": 0.6875390625, + "pass@2": 0.7691173720472443, + "pass@4": 0.8240086892575941, + "pass@8": 0.8627536267659003, + "pass@16": 0.890500855679011, + "pass@32": 0.9109176601230866, + "pass@64": 0.9261058523936584, + "pass@128": 0.938 + }, + "20": { + "pass@1": 0.608734375, + "pass@2": 0.7002310531496069, + "pass@4": 0.7665407703412074, + "pass@8": 0.8162926888982398, + "pass@16": 0.854992645216184, + "pass@32": 0.88612938304957, + "pass@64": 0.9103645543215643, + "pass@128": 0.929 + } + }, + "per_op_accuracy": { + "11": 0.9725234375, + "12": 0.9633828125, + "13": 0.9512578125, + "14": 0.9520390625, + "15": 0.959546875, + "16": 0.896828125, + "17": 0.8799609375, + "18": 0.83540625, + "19": 0.6875390625, + "20": 0.608734375 + }, + "per_op_avg_response_len": { + "11": 301.0858671875, + "12": 313.5365234375, + "13": 334.6242578125, + "14": 361.643125, + "15": 381.6772421875, + "16": 390.87915625, + "17": 412.801421875, + "18": 429.3104140625, + "19": 428.1585078125, + "20": 444.570578125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 301.0858671875, + "12": 313.5365234375, + "13": 334.6242578125, + "14": 361.643125, + "15": 381.6772421875, + "16": 390.87915625, + "17": 412.801421875, + "18": 429.3104140625, + "19": 428.1585078125, + "20": 444.570578125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 371057, + "answer_accuracy": 0.8614807763744428, + "avg_response_len": 361.376033153789, + "resp_tokens_sum": 155651885, + "pass_at_k": { + "pass@1": 0.8614807763744428, + "pass@2": 0.8980265601783103, + "pass@4": 0.9228763205676597, + "pass@8": 0.9399765617471983, + "pass@16": 0.9521830805726366, + "pass@32": 0.9618381392489881, + "pass@64": 0.9695388585332727, + "pass@128": 0.975334323922734 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9682852337110481, + "pass@2": 0.977452822823493, + "pass@4": 0.9842284054082472, + "pass@8": 0.9891870968251745, + "pass@16": 0.9931120473781875, + "pass@32": 0.9965309204049758, + "pass@64": 0.9991380576749798, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9647775423728814, + "pass@2": 0.9793001245607014, + "pass@4": 0.9885038096403199, + "pass@8": 0.9930489687895803, + "pass@16": 0.9957842683111874, + "pass@32": 0.9978504659297166, + "pass@64": 0.9992600866784821, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9408797021943573, + "pass@2": 0.9649356379433759, + "pass@4": 0.9808571087704949, + "pass@8": 0.9881229406714634, + "pass@16": 0.99055864455524, + "pass@32": 0.9913680560096522, + "pass@64": 0.9921630087220765, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.9360640576323987, + "pass@2": 0.9562122856722352, + "pass@4": 0.9707606814919159, + "pass@8": 0.9804801905602375, + "pass@16": 0.9866715910082771, + "pass@32": 0.9909012287981317, + "pass@64": 0.9945214531172304, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9660404624277457, + "pass@2": 0.9819844198033775, + "pass@4": 0.989337811097312, + "pass@8": 0.9927511350152156, + "pass@16": 0.9945051007958996, + "pass@32": 0.9954700452049894, + "pass@64": 0.996392965878996, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8782762096774194, + "pass@2": 0.9086745142240289, + "pass@4": 0.9307146733674418, + "pass@8": 0.9456327971297461, + "pass@16": 0.9552193362938507, + "pass@32": 0.9633343766463572, + "pass@64": 0.9715414297733572, + "pass@128": 0.9806451612903225 + }, + "17": { + "pass@1": 0.8934248554913294, + "pass@2": 0.9237297175822676, + "pass@4": 0.9422482222236664, + "pass@8": 0.9540871391786083, + "pass@16": 0.9620315216397886, + "pass@32": 0.9683743703109818, + "pass@64": 0.9725996832538545, + "pass@128": 0.9739884393063584 + }, + "18": { + "pass@1": 0.8005357142857142, + "pass@2": 0.8531492547806522, + "pass@4": 0.8860468795864797, + "pass@8": 0.9091291155842222, + "pass@16": 0.9275554499538204, + "pass@32": 0.9442031158786668, + "pass@64": 0.9575396478067062, + "pass@128": 0.9657142857142857 + }, + "19": { + "pass@1": 0.6647428678678678, + "pass@2": 0.7474041462722564, + "pass@4": 0.8052284390689899, + "pass@8": 0.8453281707689558, + "pass@16": 0.8732022792173314, + "pass@32": 0.8947298964394246, + "pass@64": 0.9117678285429259, + "pass@128": 0.924924924924925 + }, + "20": { + "pass@1": 0.5938203828828829, + "pass@2": 0.6811821664183864, + "pass@4": 0.7462476041846112, + "pass@8": 0.7985336624704658, + "pass@16": 0.8402692553045812, + "pass@32": 0.8729885399962004, + "pass@64": 0.8982468595231101, + "pass@128": 0.918918918918919 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 378048, + "answer_accuracy": 0.8782337198929527, + "avg_response_len": 375.42510175066906, + "resp_tokens_sum": 161606991, + "pass_at_k": { + "pass@1": 0.8782337198929527, + "pass@2": 0.9133105372031476, + "pass@4": 0.936427513771977, + "pass@8": 0.9522564458294169, + "pass@16": 0.9631596491448439, + "pass@32": 0.9708932145154232, + "pass@64": 0.9767694340645282, + "pass@128": 0.9815640796907523 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9791274472891566, + "pass@2": 0.9844943257281095, + "pass@4": 0.9885222114554959, + "pass@8": 0.9915842526776513, + "pass@16": 0.9936573596958077, + "pass@32": 0.9946784629745986, + "pass@64": 0.9954818436151764, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9556720219435737, + "pass@2": 0.9739207939426853, + "pass@4": 0.9861632180381843, + "pass@8": 0.9932283805113136, + "pass@16": 0.9967175800031935, + "pass@32": 0.9986257066354457, + "pass@64": 0.9997926928260641, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9692752100840336, + "pass@2": 0.9822620977524867, + "pass@4": 0.9899195350056031, + "pass@8": 0.9950258375460176, + "pass@16": 0.9984359616586164, + "pass@32": 0.9998247453476256, + "pass@64": 0.9999989636921891, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9544956140350878, + "pass@2": 0.9714041706957683, + "pass@4": 0.9850686652107085, + "pass@8": 0.9941410138686707, + "pass@16": 0.9982549357238066, + "pass@32": 0.9997482243275714, + "pass@64": 0.9999976075223646, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.962405303030303, + "pass@2": 0.9801527827487472, + "pass@4": 0.9897256777561895, + "pass@8": 0.9949554819709671, + "pass@16": 0.9975806792718727, + "pass@32": 0.9989957428151091, + "pass@64": 0.999819329898887, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9157540954415955, + "pass@2": 0.9383107460125177, + "pass@4": 0.9528681665860143, + "pass@8": 0.9618255529076115, + "pass@16": 0.9666002487818389, + "pass@32": 0.9691287175972142, + "pass@64": 0.9713401996282022, + "pass@128": 0.9743589743589743 + }, + "17": { + "pass@1": 0.8866459627329193, + "pass@2": 0.9140307869125057, + "pass@4": 0.9320510835796145, + "pass@8": 0.9470251241563546, + "pass@16": 0.958828933724438, + "pass@32": 0.9669829599693496, + "pass@64": 0.9726542282720683, + "pass@128": 0.9782608695652174 + }, + "18": { + "pass@1": 0.8592830882352941, + "pass@2": 0.9078276980083376, + "pass@4": 0.9323671411477975, + "pass@8": 0.9477004466579106, + "pass@16": 0.9584144161429146, + "pass@32": 0.9654229587125491, + "pass@64": 0.9701825020374101, + "pass@128": 0.9735294117647059 + }, + "19": { + "pass@1": 0.6903366815476191, + "pass@2": 0.7712544232752157, + "pass@4": 0.8257569664841, + "pass@8": 0.8643543757323078, + "pass@16": 0.8929841094671906, + "pass@32": 0.914312749232494, + "pass@64": 0.9313348502509535, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.6061938622754491, + "pass@2": 0.7069592360554483, + "pass@4": 0.779909319718268, + "pass@8": 0.8312956919046384, + "pass@16": 0.8691291374054729, + "pass@32": 0.9006392983887458, + "pass@64": 0.926877074969234, + "pass@128": 0.9461077844311377 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 365419, + "answer_accuracy": 0.8725048708740831, + "avg_response_len": 403.3319452933985, + "resp_tokens_sum": 168921872, + "pass_at_k": { + "pass@1": 0.8725048708740831, + "pass@2": 0.9097931341205792, + "pass@4": 0.9338442681389322, + "pass@8": 0.9498715941160362, + "pass@16": 0.9610604404675287, + "pass@32": 0.9693518553783383, + "pass@64": 0.9752188918576026, + "pass@128": 0.9795232273838631 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9703125, + "pass@2": 0.9826138920134985, + "pass@4": 0.9905110777819439, + "pass@8": 0.9952361673715535, + "pass@16": 0.9982486278795625, + "pass@32": 0.9997469296363813, + "pass@64": 0.9999980432330696, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9693950688073395, + "pass@2": 0.9864906149919334, + "pass@4": 0.9964843422668495, + "pass@8": 0.999587940904666, + "pass@16": 0.9999840639525752, + "pass@32": 0.9999999728189605, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9416232638888888, + "pass@2": 0.9657007934772044, + "pass@4": 0.9807125428765847, + "pass@8": 0.9889479333237797, + "pass@16": 0.9935783281704044, + "pass@32": 0.9965795260843454, + "pass@64": 0.9983950138936485, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9647626112759644, + "pass@2": 0.9797271840463564, + "pass@4": 0.988019856104782, + "pass@8": 0.992716483465879, + "pass@16": 0.9955701659139958, + "pass@32": 0.9973444053304108, + "pass@64": 0.9984967309084293, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9497010030864198, + "pass@2": 0.9660535597112858, + "pass@4": 0.9750617191369597, + "pass@8": 0.9813197913201593, + "pass@16": 0.9856122764931645, + "pass@32": 0.9887173282223192, + "pass@64": 0.991433089630488, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8941970870206489, + "pass@2": 0.9212493177014375, + "pass@4": 0.9393702139555561, + "pass@8": 0.9519523737165061, + "pass@16": 0.9612349386592156, + "pass@32": 0.9691753292432682, + "pass@64": 0.9747878703756769, + "pass@128": 0.976401179941003 + }, + "17": { + "pass@1": 0.8594455948795181, + "pass@2": 0.897720433900958, + "pass@4": 0.922174053073938, + "pass@8": 0.9382293341864976, + "pass@16": 0.9493378639982005, + "pass@32": 0.959124887388815, + "pass@64": 0.9681289020289342, + "pass@128": 0.9759036144578314 + }, + "18": { + "pass@1": 0.8485887096774194, + "pass@2": 0.903131746888494, + "pass@4": 0.9370043603420539, + "pass@8": 0.9578532946585311, + "pass@16": 0.9702169368171149, + "pass@32": 0.9777537348282815, + "pass@64": 0.9826360654372754, + "pass@128": 0.9870967741935484 + }, + "19": { + "pass@1": 0.7076331193353474, + "pass@2": 0.7887924625924787, + "pass@4": 0.8411277290300948, + "pass@8": 0.878659443062788, + "pass@16": 0.9053831900865986, + "pass@32": 0.9237568606242907, + "pass@64": 0.9352225250892545, + "pass@128": 0.9425981873111783 + }, + "20": { + "pass@1": 0.626196509009009, + "pass@2": 0.7125315522215124, + "pass@4": 0.7734252413268156, + "pass@8": 0.8190036582565382, + "pass@16": 0.8555370910100033, + "pass@32": 0.8847167374444287, + "pass@64": 0.9059201413232556, + "pass@128": 0.9219219219219219 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9286879111842106, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9286879111842106, + "pass@2": 0.9486924406858649, + "pass@4": 0.9619633238756299, + "pass@8": 0.970989572117595, + "pass@16": 0.9773331856615709, + "pass@32": 0.9821242677264577, + "pass@64": 0.985816159272922, + "pass@128": 0.9886842105263158 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9540625, + "pass@2": 0.9639723179133853, + "pass@4": 0.9723782580614919, + "pass@8": 0.9787925265522301, + "pass@16": 0.9837687276335041, + "pass@32": 0.9881571311082412, + "pass@64": 0.9927600328966237, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99946875, + "pass@2": 0.9998220964566928, + "pass@4": 0.9999744520997375, + "pass@8": 0.9999994736310946, + "pass@16": 0.9999999998889773, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999890625, + "pass@2": 0.9999948326771654, + "pass@4": 0.9999999880952382, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9944765625, + "pass@2": 0.9970152559055121, + "pass@4": 0.9982060622422197, + "pass@8": 0.9988426718080923, + "pass@16": 0.9993716905884307, + "pass@32": 0.9998027430992852, + "pass@64": 0.9999860378787945, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999140625, + "pass@2": 0.9999985236220473, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99834375, + "pass@2": 0.9991849163385828, + "pass@4": 0.9996791599175103, + "pass@8": 0.9999399336710925, + "pass@16": 0.9999976797361912, + "pass@32": 0.9999999983745138, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9961796875, + "pass@2": 0.9982639025590552, + "pass@4": 0.9995023362392201, + "pass@8": 0.999940916408823, + "pass@16": 0.9999986572834463, + "pass@32": 0.9999999995728239, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.995515625, + "pass@2": 0.9967386811023622, + "pass@4": 0.9973399041994752, + "pass@8": 0.9978469346115266, + "pass@16": 0.9984020450287275, + "pass@32": 0.9989827248615993, + "pass@64": 0.9994708755285203, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9725234375, + "pass@2": 0.9814163385826772, + "pass@4": 0.9876329908136482, + "pass@8": 0.9918884097903063, + "pass@16": 0.9949111139255706, + "pass@32": 0.9969289474459831, + "pass@64": 0.9981950900579236, + "pass@128": 0.999 + }, + "12": { + "pass@1": 0.9633828125, + "pass@2": 0.9799354084645666, + "pass@4": 0.9903667950881146, + "pass@8": 0.9952444450104463, + "pass@16": 0.9974553279156712, + "pass@32": 0.9988006564676271, + "pass@64": 0.9996719396956972, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9512578125, + "pass@2": 0.9713690944881891, + "pass@4": 0.9840455555868022, + "pass@8": 0.9908545724750295, + "pass@16": 0.9943492242524582, + "pass@32": 0.9960756104075091, + "pass@64": 0.9969796143219964, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.9520390625, + "pass@2": 0.9693324311023619, + "pass@4": 0.9814703537682793, + "pass@8": 0.9892758228409231, + "pass@16": 0.9936319146442153, + "pass@32": 0.9960982517605781, + "pass@64": 0.9977339665394206, + "pass@128": 0.999 + }, + "15": { + "pass@1": 0.959546875, + "pass@2": 0.9762183809055123, + "pass@4": 0.9848403532995874, + "pass@8": 0.9897748141534157, + "pass@16": 0.9926387666188844, + "pass@32": 0.9944456451139438, + "pass@64": 0.9959166661010433, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.896828125, + "pass@2": 0.9233396899606302, + "pass@4": 0.9414247777465318, + "pass@8": 0.953458790870688, + "pass@16": 0.9612533257789929, + "pass@32": 0.9673482732504611, + "pass@64": 0.9725713413565941, + "pass@128": 0.977 + }, + "17": { + "pass@1": 0.8799609375, + "pass@2": 0.9119715797244108, + "pass@4": 0.9323001194225726, + "pass@8": 0.9465483790840621, + "pass@16": 0.9567859939940381, + "pass@32": 0.9648555078508166, + "pass@64": 0.9711329473830459, + "pass@128": 0.976 + }, + "18": { + "pass@1": 0.83540625, + "pass@2": 0.8872344980314973, + "pass@4": 0.9175925875515554, + "pass@8": 0.9373478636623114, + "pass@16": 0.9512725593857325, + "pass@32": 0.9618185543165673, + "pass@64": 0.9696181077106221, + "pass@128": 0.975 + }, + "19": { + "pass@1": 0.6875390625, + "pass@2": 0.7691173720472443, + "pass@4": 0.8240086892575941, + "pass@8": 0.8627536267659003, + "pass@16": 0.890500855679011, + "pass@32": 0.9109176601230866, + "pass@64": 0.9261058523936584, + "pass@128": 0.938 + }, + "20": { + "pass@1": 0.608734375, + "pass@2": 0.7002310531496069, + "pass@4": 0.7665407703412074, + "pass@8": 0.8162926888982398, + "pass@16": 0.854992645216184, + "pass@32": 0.88612938304957, + "pass@64": 0.9103645543215643, + "pass@128": 0.929 + } + }, + "per_op_accuracy": { + "10": 0.9540625, + "2": 1.0, + "3": 0.99946875, + "4": 0.999890625, + "5": 0.9944765625, + "6": 0.9999140625, + "7": 0.99834375, + "8": 0.9961796875, + "9": 0.995515625, + "11": 0.9725234375, + "12": 0.9633828125, + "13": 0.9512578125, + "14": 0.9520390625, + "15": 0.959546875, + "16": 0.896828125, + "17": 0.8799609375, + "18": 0.83540625, + "19": 0.6875390625, + "20": 0.608734375 + }, + "per_op_avg_response_len": { + "10": 281.6685703125, + "2": 115.6646328125, + "3": 144.9749609375, + "4": 159.97440625, + "5": 173.51565625, + "6": 198.98446875, + "7": 224.252796875, + "8": 243.7776015625, + "9": 259.75646875, + "11": 301.0858671875, + "12": 313.5365234375, + "13": 334.6242578125, + "14": 361.643125, + "15": 381.6772421875, + "16": 390.87915625, + "17": 412.801421875, + "18": 429.3104140625, + "19": 428.1585078125, + "20": 444.570578125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.6685703125, + "2": 115.6646328125, + "3": 144.9749609375, + "4": 159.97440625, + "5": 173.51565625, + "6": 198.98446875, + "7": 224.252796875, + "8": 243.7776015625, + "9": 259.75646875, + "11": 301.0858671875, + "12": 313.5365234375, + "13": 334.6242578125, + "14": 361.643125, + "15": 381.6772421875, + "16": 390.87915625, + "17": 412.801421875, + "18": 429.3104140625, + "19": 428.1585078125, + "20": 444.570578125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 747714, + "answer_accuracy": 0.9234137883338602, + "avg_response_len": 281.5405074296554, + "resp_tokens_sum": 227971232, + "pass_at_k": { + "pass@1": 0.9234137883338602, + "pass@2": 0.9435549933283615, + "pass@4": 0.9572698152890534, + "pass@8": 0.9667895172243828, + "pass@16": 0.9736588935734707, + "pass@32": 0.9790904995462497, + "pass@64": 0.9834172709208217, + "pass@128": 0.9867214669617452 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954484697640118, + "pass@2": 0.9629707860079438, + "pass@4": 0.9709464638269774, + "pass@8": 0.9782191819839355, + "pass@16": 0.984593484638989, + "pass@32": 0.9892985145351701, + "pass@64": 0.9930035863666121, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9980902777777778, + "pass@2": 0.9997808867641542, + "pass@4": 0.9999950205629058, + "pass@8": 0.9999999974305525, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 0.9999977898533642, + "pass@4": 0.9999999997193464, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9991910133136095, + "pass@2": 0.9999253803056423, + "pass@4": 0.9999992966855474, + "pass@8": 0.9999999999707041, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9977964743589743, + "pass@2": 0.9990386192711489, + "pass@4": 0.9997245203123646, + "pass@8": 0.9999788364839716, + "pass@16": 0.9999999148409177, + "pass@32": 0.9999999999998113, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962367957746479, + "pass@2": 0.9972319646223802, + "pass@4": 0.9975970072402921, + "pass@8": 0.9979708642982112, + "pass@16": 0.9985716335094632, + "pass@32": 0.9993491975474896, + "pass@64": 0.9999188200066541, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9682852337110481, + "pass@2": 0.977452822823493, + "pass@4": 0.9842284054082472, + "pass@8": 0.9891870968251745, + "pass@16": 0.9931120473781875, + "pass@32": 0.9965309204049758, + "pass@64": 0.9991380576749798, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9647775423728814, + "pass@2": 0.9793001245607014, + "pass@4": 0.9885038096403199, + "pass@8": 0.9930489687895803, + "pass@16": 0.9957842683111874, + "pass@32": 0.9978504659297166, + "pass@64": 0.9992600866784821, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9408797021943573, + "pass@2": 0.9649356379433759, + "pass@4": 0.9808571087704949, + "pass@8": 0.9881229406714634, + "pass@16": 0.99055864455524, + "pass@32": 0.9913680560096522, + "pass@64": 0.9921630087220765, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.9360640576323987, + "pass@2": 0.9562122856722352, + "pass@4": 0.9707606814919159, + "pass@8": 0.9804801905602375, + "pass@16": 0.9866715910082771, + "pass@32": 0.9909012287981317, + "pass@64": 0.9945214531172304, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9660404624277457, + "pass@2": 0.9819844198033775, + "pass@4": 0.989337811097312, + "pass@8": 0.9927511350152156, + "pass@16": 0.9945051007958996, + "pass@32": 0.9954700452049894, + "pass@64": 0.996392965878996, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8782762096774194, + "pass@2": 0.9086745142240289, + "pass@4": 0.9307146733674418, + "pass@8": 0.9456327971297461, + "pass@16": 0.9552193362938507, + "pass@32": 0.9633343766463572, + "pass@64": 0.9715414297733572, + "pass@128": 0.9806451612903225 + }, + "17": { + "pass@1": 0.8934248554913294, + "pass@2": 0.9237297175822676, + "pass@4": 0.9422482222236664, + "pass@8": 0.9540871391786083, + "pass@16": 0.9620315216397886, + "pass@32": 0.9683743703109818, + "pass@64": 0.9725996832538545, + "pass@128": 0.9739884393063584 + }, + "18": { + "pass@1": 0.8005357142857142, + "pass@2": 0.8531492547806522, + "pass@4": 0.8860468795864797, + "pass@8": 0.9091291155842222, + "pass@16": 0.9275554499538204, + "pass@32": 0.9442031158786668, + "pass@64": 0.9575396478067062, + "pass@128": 0.9657142857142857 + }, + "19": { + "pass@1": 0.6647428678678678, + "pass@2": 0.7474041462722564, + "pass@4": 0.8052284390689899, + "pass@8": 0.8453281707689558, + "pass@16": 0.8732022792173314, + "pass@32": 0.8947298964394246, + "pass@64": 0.9117678285429259, + "pass@128": 0.924924924924925 + }, + "20": { + "pass@1": 0.5938203828828829, + "pass@2": 0.6811821664183864, + "pass@4": 0.7462476041846112, + "pass@8": 0.7985336624704658, + "pass@16": 0.8402692553045812, + "pass@32": 0.8729885399962004, + "pass@64": 0.8982468595231101, + "pass@128": 0.918918918918919 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 765697, + "answer_accuracy": 0.9325031664068589, + "avg_response_len": 291.1401585639127, + "resp_tokens_sum": 239061007, + "pass_at_k": { + "pass@1": 0.9325031664068589, + "pass@2": 0.9518363817271153, + "pass@4": 0.9646200336464147, + "pass@8": 0.9733466730553039, + "pass@16": 0.9793728174194902, + "pass@32": 0.983754479786111, + "pass@64": 0.9872180069196503, + "pass@128": 0.9900233826968043 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9433713573619632, + "pass@2": 0.9535782540698516, + "pass@4": 0.9618546829460732, + "pass@8": 0.968359742151891, + "pass@16": 0.9741511694787096, + "pass@32": 0.9805862212217205, + "pass@64": 0.9881224160445984, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983901515151515, + "pass@2": 0.999460898353615, + "pass@4": 0.9999225821204167, + "pass@8": 0.9999984049427112, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999813, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935914855072464, + "pass@2": 0.9972134257674314, + "pass@4": 0.9991986126734157, + "pass@8": 0.9998860811105759, + "pass@16": 0.9999969943734291, + "pass@32": 0.9999999992176973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999328438395415, + "pass@2": 0.99999894242267, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981484826589595, + "pass@2": 0.9991423353511447, + "pass@4": 0.9997656553768929, + "pass@8": 0.9999830971478822, + "pass@16": 0.9999999407613452, + "pass@32": 0.9999999999999076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9978213028169014, + "pass@2": 0.998696212709327, + "pass@4": 0.9994122448602376, + "pass@8": 0.9998848700630322, + "pass@16": 0.9999964254002791, + "pass@32": 0.9999999987973975, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9976609281437125, + "pass@2": 0.9988149930454053, + "pass@4": 0.9995776498184733, + "pass@8": 0.9999453661202221, + "pass@16": 0.9999993082440582, + "pass@32": 0.9999999999733661, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9791274472891566, + "pass@2": 0.9844943257281095, + "pass@4": 0.9885222114554959, + "pass@8": 0.9915842526776513, + "pass@16": 0.9936573596958077, + "pass@32": 0.9946784629745986, + "pass@64": 0.9954818436151764, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9556720219435737, + "pass@2": 0.9739207939426853, + "pass@4": 0.9861632180381843, + "pass@8": 0.9932283805113136, + "pass@16": 0.9967175800031935, + "pass@32": 0.9986257066354457, + "pass@64": 0.9997926928260641, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9692752100840336, + "pass@2": 0.9822620977524867, + "pass@4": 0.9899195350056031, + "pass@8": 0.9950258375460176, + "pass@16": 0.9984359616586164, + "pass@32": 0.9998247453476256, + "pass@64": 0.9999989636921891, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9544956140350878, + "pass@2": 0.9714041706957683, + "pass@4": 0.9850686652107085, + "pass@8": 0.9941410138686707, + "pass@16": 0.9982549357238066, + "pass@32": 0.9997482243275714, + "pass@64": 0.9999976075223646, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.962405303030303, + "pass@2": 0.9801527827487472, + "pass@4": 0.9897256777561895, + "pass@8": 0.9949554819709671, + "pass@16": 0.9975806792718727, + "pass@32": 0.9989957428151091, + "pass@64": 0.999819329898887, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9157540954415955, + "pass@2": 0.9383107460125177, + "pass@4": 0.9528681665860143, + "pass@8": 0.9618255529076115, + "pass@16": 0.9666002487818389, + "pass@32": 0.9691287175972142, + "pass@64": 0.9713401996282022, + "pass@128": 0.9743589743589743 + }, + "17": { + "pass@1": 0.8866459627329193, + "pass@2": 0.9140307869125057, + "pass@4": 0.9320510835796145, + "pass@8": 0.9470251241563546, + "pass@16": 0.958828933724438, + "pass@32": 0.9669829599693496, + "pass@64": 0.9726542282720683, + "pass@128": 0.9782608695652174 + }, + "18": { + "pass@1": 0.8592830882352941, + "pass@2": 0.9078276980083376, + "pass@4": 0.9323671411477975, + "pass@8": 0.9477004466579106, + "pass@16": 0.9584144161429146, + "pass@32": 0.9654229587125491, + "pass@64": 0.9701825020374101, + "pass@128": 0.9735294117647059 + }, + "19": { + "pass@1": 0.6903366815476191, + "pass@2": 0.7712544232752157, + "pass@4": 0.8257569664841, + "pass@8": 0.8643543757323078, + "pass@16": 0.8929841094671906, + "pass@32": 0.914312749232494, + "pass@64": 0.9313348502509535, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.6061938622754491, + "pass@2": 0.7069592360554483, + "pass@4": 0.779909319718268, + "pass@8": 0.8312956919046384, + "pass@16": 0.8691291374054729, + "pass@32": 0.9006392983887458, + "pass@64": 0.926877074969234, + "pass@128": 0.9461077844311377 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 745158, + "answer_accuracy": 0.930108144272248, + "avg_response_len": 311.89763365753316, + "resp_tokens_sum": 249877413, + "pass_at_k": { + "pass@1": 0.930108144272248, + "pass@2": 0.9506625813159659, + "pass@4": 0.9639841486302464, + "pass@8": 0.9728187372779841, + "pass@16": 0.9789563417603896, + "pass@32": 0.9835196674940303, + "pass@64": 0.9868039392795732, + "pass@128": 0.9892954146029718 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9640391791044776, + "pass@2": 0.9751006287460335, + "pass@4": 0.9840680005484389, + "pass@8": 0.989525217666148, + "pass@16": 0.9922932987785924, + "pass@32": 0.9943696315299635, + "pass@64": 0.9970265954859788, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998857821637427, + "pass@2": 0.9999978415527007, + "pass@4": 0.9999999997259115, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920266544117647, + "pass@2": 0.9942518961324689, + "pass@4": 0.9955414982134585, + "pass@8": 0.9967116901009395, + "pass@16": 0.998155080969405, + "pass@32": 0.9994198334387632, + "pass@64": 0.9999589349376307, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.9999988356640751, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976513053797469, + "pass@2": 0.9984395245689226, + "pass@4": 0.9992420280930073, + "pass@8": 0.9998284242652762, + "pass@16": 0.9999927222555876, + "pass@32": 0.9999999948561573, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9929147897897898, + "pass@2": 0.9970771706746117, + "pass@4": 0.9993902071362702, + "pass@8": 0.9999651367971397, + "pass@16": 0.9999998583663093, + "pass@32": 0.9999999999994196, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9923884646302251, + "pass@2": 0.9939457427146365, + "pass@4": 0.994643188391644, + "pass@8": 0.995451850487162, + "pass@16": 0.9964930745316808, + "pass@32": 0.9974718962737498, + "pass@64": 0.9983913004056533, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9703125, + "pass@2": 0.9826138920134985, + "pass@4": 0.9905110777819439, + "pass@8": 0.9952361673715535, + "pass@16": 0.9982486278795625, + "pass@32": 0.9997469296363813, + "pass@64": 0.9999980432330696, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9693950688073395, + "pass@2": 0.9864906149919334, + "pass@4": 0.9964843422668495, + "pass@8": 0.999587940904666, + "pass@16": 0.9999840639525752, + "pass@32": 0.9999999728189605, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9416232638888888, + "pass@2": 0.9657007934772044, + "pass@4": 0.9807125428765847, + "pass@8": 0.9889479333237797, + "pass@16": 0.9935783281704044, + "pass@32": 0.9965795260843454, + "pass@64": 0.9983950138936485, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9647626112759644, + "pass@2": 0.9797271840463564, + "pass@4": 0.988019856104782, + "pass@8": 0.992716483465879, + "pass@16": 0.9955701659139958, + "pass@32": 0.9973444053304108, + "pass@64": 0.9984967309084293, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9497010030864198, + "pass@2": 0.9660535597112858, + "pass@4": 0.9750617191369597, + "pass@8": 0.9813197913201593, + "pass@16": 0.9856122764931645, + "pass@32": 0.9887173282223192, + "pass@64": 0.991433089630488, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8941970870206489, + "pass@2": 0.9212493177014375, + "pass@4": 0.9393702139555561, + "pass@8": 0.9519523737165061, + "pass@16": 0.9612349386592156, + "pass@32": 0.9691753292432682, + "pass@64": 0.9747878703756769, + "pass@128": 0.976401179941003 + }, + "17": { + "pass@1": 0.8594455948795181, + "pass@2": 0.897720433900958, + "pass@4": 0.922174053073938, + "pass@8": 0.9382293341864976, + "pass@16": 0.9493378639982005, + "pass@32": 0.959124887388815, + "pass@64": 0.9681289020289342, + "pass@128": 0.9759036144578314 + }, + "18": { + "pass@1": 0.8485887096774194, + "pass@2": 0.903131746888494, + "pass@4": 0.9370043603420539, + "pass@8": 0.9578532946585311, + "pass@16": 0.9702169368171149, + "pass@32": 0.9777537348282815, + "pass@64": 0.9826360654372754, + "pass@128": 0.9870967741935484 + }, + "19": { + "pass@1": 0.7076331193353474, + "pass@2": 0.7887924625924787, + "pass@4": 0.8411277290300948, + "pass@8": 0.878659443062788, + "pass@16": 0.9053831900865986, + "pass@32": 0.9237568606242907, + "pass@64": 0.9352225250892545, + "pass@128": 0.9425981873111783 + }, + "20": { + "pass@1": 0.626196509009009, + "pass@2": 0.7125315522215124, + "pass@4": 0.7734252413268156, + "pass@8": 0.8190036582565382, + "pass@16": 0.8555370910100033, + "pass@32": 0.8847167374444287, + "pass@64": 0.9059201413232556, + "pass@128": 0.9219219219219219 + } + } + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e44ebd25d50ed4a646c8a9ca6772ccedd9c5df81 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a5925978851386faf56119b57768e1caa74f21607b7ec7511375d93471a972 +size 373196229 diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1021f59c488b34e51d52848a4cf9bf6b2d7b66b7 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e985305d3fdfee25237f92c09828b1109fb641a1a34130bd6648ed3e186799e1 +size 422225883 diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/composition_score_report.jsonl b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6425a4c65d865c05a890217cec59939a8e0f1f32 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition", "results_dir": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024488, "value_mismatch": 8666, "dependency_mismatch": 126198, "answer_mismatch": 7955, "extra_nodes": 11048, "missing_nodes": 8581, "perfect_with_extra": 938, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337887, "dependency_mismatch": 40758, "extra_nodes": 3484, "missing_nodes": 2571, "value_mismatch": 2633, "answer_mismatch": 2351, "perfect_with_extra": 420}, "teachers_in_school": {"total": 390656, "perfect": 346858, "value_mismatch": 3143, "answer_mismatch": 3007, "dependency_mismatch": 43394, "extra_nodes": 3920, "missing_nodes": 3330, "perfect_with_extra": 326}, "movie_festival_awards": {"total": 382336, "perfect": 339743, "dependency_mismatch": 42046, "extra_nodes": 3644, "missing_nodes": 2680, "perfect_with_extra": 192, "value_mismatch": 2890, "answer_mismatch": 2597}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 645759, "value_mismatch": 214993, "dependency_mismatch": 610706, "answer_mismatch": 165476, "extra_nodes": 40541, "missing_nodes": 147475, "perfect_with_extra": 294, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 215781, "value_mismatch": 75701, "dependency_mismatch": 207056, "answer_mismatch": 59663, "extra_nodes": 15260, "perfect_with_extra": 59, "missing_nodes": 55087}, "movie_festival_awards": {"total": 418816, "perfect": 212276, "value_mismatch": 70597, "dependency_mismatch": 198554, "answer_mismatch": 53397, "missing_nodes": 45668, "extra_nodes": 12686, "perfect_with_extra": 95}, "teachers_in_school": {"total": 430464, "perfect": 217702, "value_mismatch": 68695, "dependency_mismatch": 205096, "answer_mismatch": 52416, "extra_nodes": 12595, "perfect_with_extra": 140, "missing_nodes": 46720}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1670247, "value_mismatch": 223659, "dependency_mismatch": 736904, "answer_mismatch": 173431, "extra_nodes": 51589, "missing_nodes": 156056, "perfect_with_extra": 1232, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 553668, "dependency_mismatch": 247814, "extra_nodes": 18744, "missing_nodes": 57658, "value_mismatch": 78334, "answer_mismatch": 62014, "perfect_with_extra": 479}, "teachers_in_school": {"total": 821120, "perfect": 564560, "value_mismatch": 71838, "answer_mismatch": 55423, "dependency_mismatch": 248490, "extra_nodes": 16515, "missing_nodes": 50050, "perfect_with_extra": 466}, "movie_festival_awards": {"total": 801152, "perfect": 552019, "dependency_mismatch": 240600, "extra_nodes": 16330, "missing_nodes": 48348, "perfect_with_extra": 287, "value_mismatch": 73487, "answer_mismatch": 55994}}}} diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/results.json b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..d2de0880fefeb14b94d5edebf3f9dc3c38252cf4 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9587301587301588, + "pass@2": 0.9809523809523809, + "pass@4": 0.9936507936507937, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9550898203592815, + "pass@2": 0.9640718562874252, + "pass@4": 0.9940119760479041, + "pass@8": 0.9940119760479041, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.893491124260355, + "pass@2": 0.9230769230769231, + "pass@4": 0.9644970414201184, + "pass@8": 0.985207100591716, + "pass@16": 0.9940828402366864, + "pass@32": 0.9970414201183432, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8301282051282052, + "pass@2": 0.8782051282051282, + "pass@4": 0.9423076923076923, + "pass@8": 0.9807692307692307, + "pass@16": 0.9935897435897436, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7633802816901408, + "pass@2": 0.8338028169014085, + "pass@4": 0.9070422535211268, + "pass@8": 0.9690140845070423, + "pass@16": 0.9887323943661972, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6371681415929203, + "pass@2": 0.7109144542772862, + "pass@4": 0.7905604719764012, + "pass@8": 0.8584070796460177, + "pass@16": 0.9056047197640118, + "pass@32": 0.9410029498525073, + "pass@64": 0.9557522123893806, + "pass@128": 0.9616519174041298 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9587301587301588, + "6": 0.9550898203592815, + "7": 0.893491124260355, + "8": 0.8301282051282052, + "9": 0.7633802816901408, + "10": 0.6371681415929203 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9809523809523809, + "6": 0.9640718562874252, + "7": 0.9230769230769231, + "8": 0.8782051282051282, + "9": 0.8338028169014085, + "10": 0.7109144542772862 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9644970414201184, + "8": 0.9423076923076923, + "9": 0.9070422535211268, + "10": 0.7905604719764012 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9940119760479041, + "7": 0.985207100591716, + "8": 0.9807692307692307, + "9": 0.9690140845070423, + "10": 0.8584070796460177 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9935897435897436, + "9": 0.9887323943661972, + "10": 0.9056047197640118 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 1.0, + "10": 0.9410029498525073 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9557522123893806 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9616519174041298 + } + }, + "total_pass_at_k": { + "pass@1": 0.8899020601148261, + "pass@2": 0.9192840256670044, + "pass@4": 0.9533941236068896, + "pass@8": 0.9756838905775076, + "pass@16": 0.9864910503208375, + "pass@32": 0.9929078014184397, + "pass@64": 0.9949341438703141, + "pass@128": 0.9956095913542722 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.95, + "pass@2": 0.9735294117647059, + "pass@4": 0.9911764705882353, + "pass@8": 0.9941176470588236, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684542586750788, + "pass@2": 0.9810725552050473, + "pass@4": 0.9842271293375394, + "pass@8": 0.9936908517350158, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8575949367088608, + "pass@2": 0.9145569620253164, + "pass@4": 0.9651898734177216, + "pass@8": 0.9936708860759493, + "pass@16": 0.9968354430379747, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8168168168168168, + "pass@2": 0.8708708708708709, + "pass@4": 0.9129129129129129, + "pass@8": 0.960960960960961, + "pass@16": 0.993993993993994, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7652733118971061, + "pass@2": 0.8263665594855305, + "pass@4": 0.9035369774919614, + "pass@8": 0.9453376205787781, + "pass@16": 0.9646302250803859, + "pass@32": 0.9935691318327974, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "10": { + "pass@1": 0.6149253731343284, + "pass@2": 0.6895522388059702, + "pass@4": 0.7701492537313432, + "pass@8": 0.8716417910447761, + "pass@16": 0.9343283582089552, + "pass@32": 0.9552238805970149, + "pass@64": 0.9791044776119403, + "pass@128": 0.9880597014925373 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.95, + "6": 0.9684542586750788, + "7": 0.8575949367088608, + "8": 0.8168168168168168, + "9": 0.7652733118971061, + "10": 0.6149253731343284 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9735294117647059, + "6": 0.9810725552050473, + "7": 0.9145569620253164, + "8": 0.8708708708708709, + "9": 0.8263665594855305, + "10": 0.6895522388059702 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9911764705882353, + "6": 0.9842271293375394, + "7": 0.9651898734177216, + "8": 0.9129129129129129, + "9": 0.9035369774919614, + "10": 0.7701492537313432 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 0.9936908517350158, + "7": 0.9936708860759493, + "8": 0.960960960960961, + "9": 0.9453376205787781, + "10": 0.8716417910447761 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.993993993993994, + "9": 0.9646302250803859, + "10": 0.9343283582089552 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9935691318327974, + "10": 0.9552238805970149 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9791044776119403 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9880597014925373 + } + }, + "total_pass_at_k": { + "pass@1": 0.8865082022095748, + "pass@2": 0.9179779042517576, + "pass@4": 0.9481084700368263, + "pass@8": 0.9735520589219954, + "pass@16": 0.987612989621694, + "pass@32": 0.9939738868429863, + "pass@64": 0.9973217274857716, + "pass@128": 0.9983260796786073 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9881656804733728, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9478260869565217, + "pass@2": 0.9826086956521739, + "pass@4": 0.991304347826087, + "pass@8": 0.9971014492753624, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9856733524355301, + "pass@4": 0.9914040114613181, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8554913294797688, + "pass@2": 0.9017341040462428, + "pass@4": 0.9595375722543352, + "pass@8": 0.9826589595375722, + "pass@16": 0.9913294797687862, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.819718309859155, + "pass@2": 0.8788732394366198, + "pass@4": 0.952112676056338, + "pass@8": 0.9830985915492958, + "pass@16": 0.9971830985915493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7455089820359282, + "pass@2": 0.8023952095808383, + "pass@4": 0.8952095808383234, + "pass@8": 0.9580838323353293, + "pass@16": 0.9850299401197605, + "pass@32": 0.9910179640718563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6319018404907976, + "pass@2": 0.6932515337423313, + "pass@4": 0.7668711656441718, + "pass@8": 0.8312883435582822, + "pass@16": 0.8957055214723927, + "pass@32": 0.9478527607361963, + "pass@64": 0.9570552147239264, + "pass@128": 0.9693251533742331 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985207100591716, + "5": 0.9478260869565217, + "6": 0.9770773638968482, + "7": 0.8554913294797688, + "8": 0.819718309859155, + "9": 0.7455089820359282, + "10": 0.6319018404907976 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9826086956521739, + "6": 0.9856733524355301, + "7": 0.9017341040462428, + "8": 0.8788732394366198, + "9": 0.8023952095808383, + "10": 0.6932515337423313 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9914040114613181, + "7": 0.9595375722543352, + "8": 0.952112676056338, + "9": 0.8952095808383234, + "10": 0.7668711656441718 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9826589595375722, + "8": 0.9830985915492958, + "9": 0.9580838323353293, + "10": 0.8312883435582822 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9913294797687862, + "8": 0.9971830985915493, + "9": 0.9850299401197605, + "10": 0.8957055214723927 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9910179640718563, + "10": 0.9478527607361963 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9570552147239264 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9693251533742331 + } + }, + "total_pass_at_k": { + "pass@1": 0.8853211009174312, + "pass@2": 0.9154652686762779, + "pass@4": 0.9511795543905636, + "pass@8": 0.9728047182175622, + "pass@16": 0.9855832241153342, + "pass@32": 0.9934469200524246, + "pass@64": 0.9954128440366973, + "pass@128": 0.9967234600262124 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.989, + "pass@2": 0.994, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.952, + "pass@2": 0.979, + "pass@4": 0.992, + "pass@8": 0.997, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.967, + "pass@2": 0.977, + "pass@4": 0.99, + "pass@8": 0.995, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.869, + "pass@2": 0.913, + "pass@4": 0.963, + "pass@8": 0.987, + "pass@16": 0.994, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.822, + "pass@2": 0.876, + "pass@4": 0.936, + "pass@8": 0.975, + "pass@16": 0.995, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.758, + "pass@2": 0.821, + "pass@4": 0.902, + "pass@8": 0.958, + "pass@16": 0.98, + "pass@32": 0.995, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.628, + "pass@2": 0.698, + "pass@4": 0.776, + "pass@8": 0.854, + "pass@16": 0.912, + "pass@32": 0.948, + "pass@64": 0.964, + "pass@128": 0.973 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.989, + "5": 0.952, + "6": 0.967, + "7": 0.869, + "8": 0.822, + "9": 0.758, + "10": 0.628 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.979, + "6": 0.977, + "7": 0.913, + "8": 0.876, + "9": 0.821, + "10": 0.698 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.992, + "6": 0.99, + "7": 0.963, + "8": 0.936, + "9": 0.902, + "10": 0.776 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.997, + "6": 0.995, + "7": 0.987, + "8": 0.975, + "9": 0.958, + "10": 0.854 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.999, + "7": 0.994, + "8": 0.995, + "9": 0.98, + "10": 0.912 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 0.995, + "10": 0.948 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.999, + "10": 0.964 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.999, + "10": 0.973 + } + }, + "total_pass_at_k": { + "pass@1": 0.8872222222222222, + "pass@2": 0.9175555555555556, + "pass@4": 0.9508888888888889, + "pass@8": 0.974, + "pass@16": 0.9865555555555555, + "pass@32": 0.9934444444444445, + "pass@64": 0.9958888888888889, + "pass@128": 0.9968888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5920679886685553, + "pass@2": 0.6798866855524079, + "pass@4": 0.7648725212464589, + "pass@8": 0.8470254957507082, + "pass@16": 0.9206798866855525, + "pass@32": 0.9603399433427762, + "pass@64": 0.9801699716713881, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6384180790960452, + "pass@2": 0.6836158192090396, + "pass@4": 0.768361581920904, + "pass@8": 0.867231638418079, + "pass@16": 0.923728813559322, + "pass@32": 0.9491525423728814, + "pass@64": 0.9717514124293786, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.5862068965517241, + "pass@2": 0.6426332288401254, + "pass@4": 0.7241379310344828, + "pass@8": 0.8119122257053292, + "pass@16": 0.8746081504702194, + "pass@32": 0.9090909090909091, + "pass@64": 0.9373040752351097, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.5514018691588785, + "pass@2": 0.5856697819314641, + "pass@4": 0.6261682242990654, + "pass@8": 0.6853582554517134, + "pass@16": 0.7476635514018691, + "pass@32": 0.8473520249221184, + "pass@64": 0.9003115264797508, + "pass@128": 0.9283489096573209 + }, + "15": { + "pass@1": 0.5028901734104047, + "pass@2": 0.5404624277456648, + "pass@4": 0.5867052023121387, + "pass@8": 0.6763005780346821, + "pass@16": 0.7485549132947977, + "pass@32": 0.838150289017341, + "pass@64": 0.9104046242774566, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.4967741935483871, + "pass@2": 0.535483870967742, + "pass@4": 0.5838709677419355, + "pass@8": 0.6225806451612903, + "pass@16": 0.6838709677419355, + "pass@32": 0.7129032258064516, + "pass@64": 0.7645161290322581, + "pass@128": 0.7935483870967742 + }, + "17": { + "pass@1": 0.5433526011560693, + "pass@2": 0.5780346820809249, + "pass@4": 0.615606936416185, + "pass@8": 0.6445086705202312, + "pass@16": 0.6936416184971098, + "pass@32": 0.7312138728323699, + "pass@64": 0.7832369942196532, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.43714285714285717, + "pass@2": 0.46, + "pass@4": 0.4828571428571429, + "pass@8": 0.5142857142857142, + "pass@16": 0.5742857142857143, + "pass@32": 0.6371428571428571, + "pass@64": 0.6971428571428572, + "pass@128": 0.7714285714285715 + }, + "19": { + "pass@1": 0.34234234234234234, + "pass@2": 0.4084084084084084, + "pass@4": 0.44144144144144143, + "pass@8": 0.4804804804804805, + "pass@16": 0.5225225225225225, + "pass@32": 0.5585585585585585, + "pass@64": 0.5975975975975976, + "pass@128": 0.6276276276276276 + }, + "20": { + "pass@1": 0.27627627627627627, + "pass@2": 0.3303303303303303, + "pass@4": 0.35135135135135137, + "pass@8": 0.4084084084084084, + "pass@16": 0.44744744744744747, + "pass@32": 0.5105105105105106, + "pass@64": 0.5705705705705706, + "pass@128": 0.6246246246246246 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5920679886685553, + "12": 0.6384180790960452, + "13": 0.5862068965517241, + "14": 0.5514018691588785, + "15": 0.5028901734104047, + "16": 0.4967741935483871, + "17": 0.5433526011560693, + "18": 0.43714285714285717, + "19": 0.34234234234234234, + "20": 0.27627627627627627 + }, + "pass@2": { + "11": 0.6798866855524079, + "12": 0.6836158192090396, + "13": 0.6426332288401254, + "14": 0.5856697819314641, + "15": 0.5404624277456648, + "16": 0.535483870967742, + "17": 0.5780346820809249, + "18": 0.46, + "19": 0.4084084084084084, + "20": 0.3303303303303303 + }, + "pass@4": { + "11": 0.7648725212464589, + "12": 0.768361581920904, + "13": 0.7241379310344828, + "14": 0.6261682242990654, + "15": 0.5867052023121387, + "16": 0.5838709677419355, + "17": 0.615606936416185, + "18": 0.4828571428571429, + "19": 0.44144144144144143, + "20": 0.35135135135135137 + }, + "pass@8": { + "11": 0.8470254957507082, + "12": 0.867231638418079, + "13": 0.8119122257053292, + "14": 0.6853582554517134, + "15": 0.6763005780346821, + "16": 0.6225806451612903, + "17": 0.6445086705202312, + "18": 0.5142857142857142, + "19": 0.4804804804804805, + "20": 0.4084084084084084 + }, + "pass@16": { + "11": 0.9206798866855525, + "12": 0.923728813559322, + "13": 0.8746081504702194, + "14": 0.7476635514018691, + "15": 0.7485549132947977, + "16": 0.6838709677419355, + "17": 0.6936416184971098, + "18": 0.5742857142857143, + "19": 0.5225225225225225, + "20": 0.44744744744744747 + }, + "pass@32": { + "11": 0.9603399433427762, + "12": 0.9491525423728814, + "13": 0.9090909090909091, + "14": 0.8473520249221184, + "15": 0.838150289017341, + "16": 0.7129032258064516, + "17": 0.7312138728323699, + "18": 0.6371428571428571, + "19": 0.5585585585585585, + "20": 0.5105105105105106 + }, + "pass@64": { + "11": 0.9801699716713881, + "12": 0.9717514124293786, + "13": 0.9373040752351097, + "14": 0.9003115264797508, + "15": 0.9104046242774566, + "16": 0.7645161290322581, + "17": 0.7832369942196532, + "18": 0.6971428571428572, + "19": 0.5975975975975976, + "20": 0.5705705705705706 + }, + "pass@128": { + "11": 0.9858356940509915, + "12": 0.9774011299435028, + "13": 0.9592476489028213, + "14": 0.9283489096573209, + "15": 0.9508670520231214, + "16": 0.7935483870967742, + "17": 0.8352601156069365, + "18": 0.7714285714285715, + "19": 0.6276276276276276, + "20": 0.6246246246246246 + } + }, + "total_pass_at_k": { + "pass@1": 0.4974739970282318, + "pass@2": 0.5453194650817236, + "pass@4": 0.5955423476968796, + "pass@8": 0.6570579494799406, + "pass@16": 0.7150074294205052, + "pass@32": 0.7667161961367014, + "pass@64": 0.812481426448737, + "pass@128": 0.8466567607726597 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6, + "pass@2": 0.6666666666666666, + "pass@4": 0.7619047619047619, + "pass@8": 0.8698412698412699, + "pass@16": 0.926984126984127, + "pass@32": 0.9619047619047619, + "pass@64": 0.9682539682539683, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.636085626911315, + "pass@2": 0.709480122324159, + "pass@4": 0.7981651376146789, + "pass@8": 0.8776758409785933, + "pass@16": 0.9296636085626911, + "pass@32": 0.963302752293578, + "pass@64": 0.9755351681957186, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.5802469135802469, + "pass@2": 0.6820987654320988, + "pass@4": 0.7561728395061729, + "pass@8": 0.8240740740740741, + "pass@16": 0.8827160493827161, + "pass@32": 0.9320987654320988, + "pass@64": 0.9629629629629629, + "pass@128": 0.9722222222222222 + }, + "14": { + "pass@1": 0.5608308605341247, + "pass@2": 0.5964391691394659, + "pass@4": 0.6617210682492581, + "pass@8": 0.7388724035608308, + "pass@16": 0.8011869436201781, + "pass@32": 0.8545994065281899, + "pass@64": 0.9109792284866469, + "pass@128": 0.9376854599406528 + }, + "15": { + "pass@1": 0.5, + "pass@2": 0.5462962962962963, + "pass@4": 0.5895061728395061, + "pass@8": 0.6574074074074074, + "pass@16": 0.7253086419753086, + "pass@32": 0.7870370370370371, + "pass@64": 0.8765432098765432, + "pass@128": 0.9259259259259259 + }, + "16": { + "pass@1": 0.5368731563421829, + "pass@2": 0.584070796460177, + "pass@4": 0.6135693215339233, + "pass@8": 0.6666666666666666, + "pass@16": 0.7168141592920354, + "pass@32": 0.7699115044247787, + "pass@64": 0.8082595870206489, + "pass@128": 0.8348082595870207 + }, + "17": { + "pass@1": 0.4939759036144578, + "pass@2": 0.5120481927710844, + "pass@4": 0.536144578313253, + "pass@8": 0.5933734939759037, + "pass@16": 0.6566265060240963, + "pass@32": 0.7198795180722891, + "pass@64": 0.7740963855421686, + "pass@128": 0.8192771084337349 + }, + "18": { + "pass@1": 0.44516129032258067, + "pass@2": 0.5064516129032258, + "pass@4": 0.532258064516129, + "pass@8": 0.5645161290322581, + "pass@16": 0.6258064516129033, + "pass@32": 0.6774193548387096, + "pass@64": 0.7483870967741936, + "pass@128": 0.7935483870967742 + }, + "19": { + "pass@1": 0.3776435045317221, + "pass@2": 0.4501510574018127, + "pass@4": 0.4652567975830816, + "pass@8": 0.5105740181268882, + "pass@16": 0.5317220543806647, + "pass@32": 0.5800604229607251, + "pass@64": 0.622356495468278, + "pass@128": 0.6737160120845922 + }, + "20": { + "pass@1": 0.2912912912912913, + "pass@2": 0.34534534534534533, + "pass@4": 0.37537537537537535, + "pass@8": 0.4084084084084084, + "pass@16": 0.4624624624624625, + "pass@32": 0.5165165165165165, + "pass@64": 0.5645645645645646, + "pass@128": 0.6096096096096096 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6, + "12": 0.636085626911315, + "13": 0.5802469135802469, + "14": 0.5608308605341247, + "15": 0.5, + "16": 0.5368731563421829, + "17": 0.4939759036144578, + "18": 0.44516129032258067, + "19": 0.3776435045317221, + "20": 0.2912912912912913 + }, + "pass@2": { + "11": 0.6666666666666666, + "12": 0.709480122324159, + "13": 0.6820987654320988, + "14": 0.5964391691394659, + "15": 0.5462962962962963, + "16": 0.584070796460177, + "17": 0.5120481927710844, + "18": 0.5064516129032258, + "19": 0.4501510574018127, + "20": 0.34534534534534533 + }, + "pass@4": { + "11": 0.7619047619047619, + "12": 0.7981651376146789, + "13": 0.7561728395061729, + "14": 0.6617210682492581, + "15": 0.5895061728395061, + "16": 0.6135693215339233, + "17": 0.536144578313253, + "18": 0.532258064516129, + "19": 0.4652567975830816, + "20": 0.37537537537537535 + }, + "pass@8": { + "11": 0.8698412698412699, + "12": 0.8776758409785933, + "13": 0.8240740740740741, + "14": 0.7388724035608308, + "15": 0.6574074074074074, + "16": 0.6666666666666666, + "17": 0.5933734939759037, + "18": 0.5645161290322581, + "19": 0.5105740181268882, + "20": 0.4084084084084084 + }, + "pass@16": { + "11": 0.926984126984127, + "12": 0.9296636085626911, + "13": 0.8827160493827161, + "14": 0.8011869436201781, + "15": 0.7253086419753086, + "16": 0.7168141592920354, + "17": 0.6566265060240963, + "18": 0.6258064516129033, + "19": 0.5317220543806647, + "20": 0.4624624624624625 + }, + "pass@32": { + "11": 0.9619047619047619, + "12": 0.963302752293578, + "13": 0.9320987654320988, + "14": 0.8545994065281899, + "15": 0.7870370370370371, + "16": 0.7699115044247787, + "17": 0.7198795180722891, + "18": 0.6774193548387096, + "19": 0.5800604229607251, + "20": 0.5165165165165165 + }, + "pass@64": { + "11": 0.9682539682539683, + "12": 0.9755351681957186, + "13": 0.9629629629629629, + "14": 0.9109792284866469, + "15": 0.8765432098765432, + "16": 0.8082595870206489, + "17": 0.7740963855421686, + "18": 0.7483870967741936, + "19": 0.622356495468278, + "20": 0.5645645645645646 + }, + "pass@128": { + "11": 0.9873015873015873, + "12": 0.9877675840978594, + "13": 0.9722222222222222, + "14": 0.9376854599406528, + "15": 0.9259259259259259, + "16": 0.8348082595870207, + "17": 0.8192771084337349, + "18": 0.7935483870967742, + "19": 0.6737160120845922, + "20": 0.6096096096096096 + } + }, + "total_pass_at_k": { + "pass@1": 0.5018337408312958, + "pass@2": 0.5592909535452323, + "pass@4": 0.6081907090464548, + "pass@8": 0.6702322738386308, + "pass@16": 0.7249388753056235, + "pass@32": 0.7753667481662592, + "pass@64": 0.8202933985330073, + "pass@128": 0.8533007334963325 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6626506024096386, + "pass@2": 0.6987951807228916, + "pass@4": 0.7921686746987951, + "pass@8": 0.8524096385542169, + "pass@16": 0.927710843373494, + "pass@32": 0.963855421686747, + "pass@64": 0.9728915662650602, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.5862068965517241, + "pass@2": 0.6551724137931034, + "pass@4": 0.7492163009404389, + "pass@8": 0.8401253918495298, + "pass@16": 0.9090909090909091, + "pass@32": 0.9561128526645768, + "pass@64": 0.9780564263322884, + "pass@128": 0.9811912225705329 + }, + "13": { + "pass@1": 0.6106442577030813, + "pass@2": 0.6694677871148459, + "pass@4": 0.7675070028011205, + "pass@8": 0.8711484593837535, + "pass@16": 0.927170868347339, + "pass@32": 0.9523809523809523, + "pass@64": 0.969187675070028, + "pass@128": 0.9915966386554622 + }, + "14": { + "pass@1": 0.5760233918128655, + "pass@2": 0.631578947368421, + "pass@4": 0.6900584795321637, + "pass@8": 0.7543859649122807, + "pass@16": 0.8099415204678363, + "pass@32": 0.8859649122807017, + "pass@64": 0.9502923976608187, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.5484848484848485, + "pass@2": 0.5848484848484848, + "pass@4": 0.6090909090909091, + "pass@8": 0.6666666666666666, + "pass@16": 0.7333333333333333, + "pass@32": 0.8272727272727273, + "pass@64": 0.896969696969697, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5384615384615384, + "pass@2": 0.5641025641025641, + "pass@4": 0.5982905982905983, + "pass@8": 0.6524216524216524, + "pass@16": 0.6866096866096866, + "pass@32": 0.7606837606837606, + "pass@64": 0.8034188034188035, + "pass@128": 0.8376068376068376 + }, + "17": { + "pass@1": 0.4254658385093168, + "pass@2": 0.4472049689440994, + "pass@4": 0.5093167701863354, + "pass@8": 0.5683229813664596, + "pass@16": 0.6242236024844721, + "pass@32": 0.6956521739130435, + "pass@64": 0.7795031055900621, + "pass@128": 0.8136645962732919 + }, + "18": { + "pass@1": 0.4176470588235294, + "pass@2": 0.45, + "pass@4": 0.47941176470588237, + "pass@8": 0.5205882352941177, + "pass@16": 0.5647058823529412, + "pass@32": 0.5794117647058824, + "pass@64": 0.6823529411764706, + "pass@128": 0.7529411764705882 + }, + "19": { + "pass@1": 0.375, + "pass@2": 0.40773809523809523, + "pass@4": 0.44047619047619047, + "pass@8": 0.4851190476190476, + "pass@16": 0.5208333333333334, + "pass@32": 0.5744047619047619, + "pass@64": 0.6190476190476191, + "pass@128": 0.6755952380952381 + }, + "20": { + "pass@1": 0.2844311377245509, + "pass@2": 0.33532934131736525, + "pass@4": 0.38323353293413176, + "pass@8": 0.4281437125748503, + "pass@16": 0.47005988023952094, + "pass@32": 0.5209580838323353, + "pass@64": 0.5658682634730539, + "pass@128": 0.6287425149700598 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6626506024096386, + "12": 0.5862068965517241, + "13": 0.6106442577030813, + "14": 0.5760233918128655, + "15": 0.5484848484848485, + "16": 0.5384615384615384, + "17": 0.4254658385093168, + "18": 0.4176470588235294, + "19": 0.375, + "20": 0.2844311377245509 + }, + "pass@2": { + "11": 0.6987951807228916, + "12": 0.6551724137931034, + "13": 0.6694677871148459, + "14": 0.631578947368421, + "15": 0.5848484848484848, + "16": 0.5641025641025641, + "17": 0.4472049689440994, + "18": 0.45, + "19": 0.40773809523809523, + "20": 0.33532934131736525 + }, + "pass@4": { + "11": 0.7921686746987951, + "12": 0.7492163009404389, + "13": 0.7675070028011205, + "14": 0.6900584795321637, + "15": 0.6090909090909091, + "16": 0.5982905982905983, + "17": 0.5093167701863354, + "18": 0.47941176470588237, + "19": 0.44047619047619047, + "20": 0.38323353293413176 + }, + "pass@8": { + "11": 0.8524096385542169, + "12": 0.8401253918495298, + "13": 0.8711484593837535, + "14": 0.7543859649122807, + "15": 0.6666666666666666, + "16": 0.6524216524216524, + "17": 0.5683229813664596, + "18": 0.5205882352941177, + "19": 0.4851190476190476, + "20": 0.4281437125748503 + }, + "pass@16": { + "11": 0.927710843373494, + "12": 0.9090909090909091, + "13": 0.927170868347339, + "14": 0.8099415204678363, + "15": 0.7333333333333333, + "16": 0.6866096866096866, + "17": 0.6242236024844721, + "18": 0.5647058823529412, + "19": 0.5208333333333334, + "20": 0.47005988023952094 + }, + "pass@32": { + "11": 0.963855421686747, + "12": 0.9561128526645768, + "13": 0.9523809523809523, + "14": 0.8859649122807017, + "15": 0.8272727272727273, + "16": 0.7606837606837606, + "17": 0.6956521739130435, + "18": 0.5794117647058824, + "19": 0.5744047619047619, + "20": 0.5209580838323353 + }, + "pass@64": { + "11": 0.9728915662650602, + "12": 0.9780564263322884, + "13": 0.969187675070028, + "14": 0.9502923976608187, + "15": 0.896969696969697, + "16": 0.8034188034188035, + "17": 0.7795031055900621, + "18": 0.6823529411764706, + "19": 0.6190476190476191, + "20": 0.5658682634730539 + }, + "pass@128": { + "11": 0.9789156626506024, + "12": 0.9811912225705329, + "13": 0.9915966386554622, + "14": 0.9736842105263158, + "15": 0.9454545454545454, + "16": 0.8376068376068376, + "17": 0.8136645962732919, + "18": 0.7529411764705882, + "19": 0.6755952380952381, + "20": 0.6287425149700598 + } + }, + "total_pass_at_k": { + "pass@1": 0.5031222123104371, + "pass@2": 0.5450490633363069, + "pass@4": 0.6024382991376747, + "pass@8": 0.6645851917930419, + "pass@16": 0.7178114778471603, + "pass@32": 0.7719298245614035, + "pass@64": 0.8218852215283973, + "pass@128": 0.8581623550401427 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.618, + "pass@2": 0.682, + "pass@4": 0.773, + "pass@8": 0.856, + "pass@16": 0.925, + "pass@32": 0.962, + "pass@64": 0.974, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.621, + "pass@2": 0.683, + "pass@4": 0.772, + "pass@8": 0.862, + "pass@16": 0.921, + "pass@32": 0.956, + "pass@64": 0.975, + "pass@128": 0.982 + }, + "13": { + "pass@1": 0.593, + "pass@2": 0.665, + "pass@4": 0.75, + "pass@8": 0.837, + "pass@16": 0.896, + "pass@32": 0.932, + "pass@64": 0.957, + "pass@128": 0.975 + }, + "14": { + "pass@1": 0.563, + "pass@2": 0.605, + "pass@4": 0.66, + "pass@8": 0.727, + "pass@16": 0.787, + "pass@32": 0.863, + "pass@64": 0.921, + "pass@128": 0.947 + }, + "15": { + "pass@1": 0.517, + "pass@2": 0.557, + "pass@4": 0.595, + "pass@8": 0.667, + "pass@16": 0.736, + "pass@32": 0.818, + "pass@64": 0.895, + "pass@128": 0.941 + }, + "16": { + "pass@1": 0.525, + "pass@2": 0.562, + "pass@4": 0.599, + "pass@8": 0.648, + "pass@16": 0.696, + "pass@32": 0.749, + "pass@64": 0.793, + "pass@128": 0.823 + }, + "17": { + "pass@1": 0.489, + "pass@2": 0.514, + "pass@4": 0.555, + "pass@8": 0.603, + "pass@16": 0.659, + "pass@32": 0.716, + "pass@64": 0.779, + "pass@128": 0.823 + }, + "18": { + "pass@1": 0.433, + "pass@2": 0.471, + "pass@4": 0.497, + "pass@8": 0.532, + "pass@16": 0.587, + "pass@32": 0.63, + "pass@64": 0.708, + "pass@128": 0.772 + }, + "19": { + "pass@1": 0.365, + "pass@2": 0.422, + "pass@4": 0.449, + "pass@8": 0.492, + "pass@16": 0.525, + "pass@32": 0.571, + "pass@64": 0.613, + "pass@128": 0.659 + }, + "20": { + "pass@1": 0.284, + "pass@2": 0.337, + "pass@4": 0.37, + "pass@8": 0.415, + "pass@16": 0.46, + "pass@32": 0.516, + "pass@64": 0.567, + "pass@128": 0.621 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.618, + "12": 0.621, + "13": 0.593, + "14": 0.563, + "15": 0.517, + "16": 0.525, + "17": 0.489, + "18": 0.433, + "19": 0.365, + "20": 0.284 + }, + "pass@2": { + "11": 0.682, + "12": 0.683, + "13": 0.665, + "14": 0.605, + "15": 0.557, + "16": 0.562, + "17": 0.514, + "18": 0.471, + "19": 0.422, + "20": 0.337 + }, + "pass@4": { + "11": 0.773, + "12": 0.772, + "13": 0.75, + "14": 0.66, + "15": 0.595, + "16": 0.599, + "17": 0.555, + "18": 0.497, + "19": 0.449, + "20": 0.37 + }, + "pass@8": { + "11": 0.856, + "12": 0.862, + "13": 0.837, + "14": 0.727, + "15": 0.667, + "16": 0.648, + "17": 0.603, + "18": 0.532, + "19": 0.492, + "20": 0.415 + }, + "pass@16": { + "11": 0.925, + "12": 0.921, + "13": 0.896, + "14": 0.787, + "15": 0.736, + "16": 0.696, + "17": 0.659, + "18": 0.587, + "19": 0.525, + "20": 0.46 + }, + "pass@32": { + "11": 0.962, + "12": 0.956, + "13": 0.932, + "14": 0.863, + "15": 0.818, + "16": 0.749, + "17": 0.716, + "18": 0.63, + "19": 0.571, + "20": 0.516 + }, + "pass@64": { + "11": 0.974, + "12": 0.975, + "13": 0.957, + "14": 0.921, + "15": 0.895, + "16": 0.793, + "17": 0.779, + "18": 0.708, + "19": 0.613, + "20": 0.567 + }, + "pass@128": { + "11": 0.984, + "12": 0.982, + "13": 0.975, + "14": 0.947, + "15": 0.941, + "16": 0.823, + "17": 0.823, + "18": 0.772, + "19": 0.659, + "20": 0.621 + } + }, + "total_pass_at_k": { + "pass@1": 0.5008, + "pass@2": 0.5498, + "pass@4": 0.602, + "pass@8": 0.6639, + "pass@16": 0.7192, + "pass@32": 0.7713, + "pass@64": 0.8182, + "pass@128": 0.8527 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/summary.csv b/Qwen3-100M-midtrain-diff_ar-block4/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..4a6feb70826c5965b1329b8eb65fcd8a219a5a00 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-38646,0.9930946180555555,0.870721875,0.9286879111842106,0.0,0.0,0.0,200.28550694444445,379.828709375,1152000,1280000,2432000 diff --git a/Qwen3-100M-midtrain-diff_ar-block4/composition/summary.json b/Qwen3-100M-midtrain-diff_ar-block4/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ca9e3087b0c1a12c7d1cdacf86846878dbde6471 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block4/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-38646", + "id_acc": 0.9930946180555555, + "ood_acc": 0.870721875, + "total_acc": 0.9286879111842106, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.28550694444445, + "ood_avg_resp_len": 379.828709375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 747714, + "answer_accuracy": 0.9234137883338602, + "avg_response_len": 281.5405074296554, + "resp_tokens_sum": 227971232, + "pass_at_k": { + "pass@1": 0.9234137883338602, + "pass@2": 0.9435549933283615, + "pass@4": 0.9572698152890534, + "pass@8": 0.9667895172243828, + "pass@16": 0.9736588935734707, + "pass@32": 0.9790904995462497, + "pass@64": 0.9834172709208217, + "pass@128": 0.9867214669617452 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954484697640118, + "pass@2": 0.9629707860079438, + "pass@4": 0.9709464638269774, + "pass@8": 0.9782191819839355, + "pass@16": 0.984593484638989, + "pass@32": 0.9892985145351701, + "pass@64": 0.9930035863666121, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9980902777777778, + "pass@2": 0.9997808867641542, + "pass@4": 0.9999950205629058, + "pass@8": 0.9999999974305525, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 0.9999977898533642, + "pass@4": 0.9999999997193464, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9991910133136095, + "pass@2": 0.9999253803056423, + "pass@4": 0.9999992966855474, + "pass@8": 0.9999999999707041, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9977964743589743, + "pass@2": 0.9990386192711489, + "pass@4": 0.9997245203123646, + "pass@8": 0.9999788364839716, + "pass@16": 0.9999999148409177, + "pass@32": 0.9999999999998113, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962367957746479, + "pass@2": 0.9972319646223802, + "pass@4": 0.9975970072402921, + "pass@8": 0.9979708642982112, + "pass@16": 0.9985716335094632, + "pass@32": 0.9993491975474896, + "pass@64": 0.9999188200066541, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9682852337110481, + "pass@2": 0.977452822823493, + "pass@4": 0.9842284054082472, + "pass@8": 0.9891870968251745, + "pass@16": 0.9931120473781875, + "pass@32": 0.9965309204049758, + "pass@64": 0.9991380576749798, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9647775423728814, + "pass@2": 0.9793001245607014, + "pass@4": 0.9885038096403199, + "pass@8": 0.9930489687895803, + "pass@16": 0.9957842683111874, + "pass@32": 0.9978504659297166, + "pass@64": 0.9992600866784821, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9408797021943573, + "pass@2": 0.9649356379433759, + "pass@4": 0.9808571087704949, + "pass@8": 0.9881229406714634, + "pass@16": 0.99055864455524, + "pass@32": 0.9913680560096522, + "pass@64": 0.9921630087220765, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.9360640576323987, + "pass@2": 0.9562122856722352, + "pass@4": 0.9707606814919159, + "pass@8": 0.9804801905602375, + "pass@16": 0.9866715910082771, + "pass@32": 0.9909012287981317, + "pass@64": 0.9945214531172304, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9660404624277457, + "pass@2": 0.9819844198033775, + "pass@4": 0.989337811097312, + "pass@8": 0.9927511350152156, + "pass@16": 0.9945051007958996, + "pass@32": 0.9954700452049894, + "pass@64": 0.996392965878996, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8782762096774194, + "pass@2": 0.9086745142240289, + "pass@4": 0.9307146733674418, + "pass@8": 0.9456327971297461, + "pass@16": 0.9552193362938507, + "pass@32": 0.9633343766463572, + "pass@64": 0.9715414297733572, + "pass@128": 0.9806451612903225 + }, + "17": { + "pass@1": 0.8934248554913294, + "pass@2": 0.9237297175822676, + "pass@4": 0.9422482222236664, + "pass@8": 0.9540871391786083, + "pass@16": 0.9620315216397886, + "pass@32": 0.9683743703109818, + "pass@64": 0.9725996832538545, + "pass@128": 0.9739884393063584 + }, + "18": { + "pass@1": 0.8005357142857142, + "pass@2": 0.8531492547806522, + "pass@4": 0.8860468795864797, + "pass@8": 0.9091291155842222, + "pass@16": 0.9275554499538204, + "pass@32": 0.9442031158786668, + "pass@64": 0.9575396478067062, + "pass@128": 0.9657142857142857 + }, + "19": { + "pass@1": 0.6647428678678678, + "pass@2": 0.7474041462722564, + "pass@4": 0.8052284390689899, + "pass@8": 0.8453281707689558, + "pass@16": 0.8732022792173314, + "pass@32": 0.8947298964394246, + "pass@64": 0.9117678285429259, + "pass@128": 0.924924924924925 + }, + "20": { + "pass@1": 0.5938203828828829, + "pass@2": 0.6811821664183864, + "pass@4": 0.7462476041846112, + "pass@8": 0.7985336624704658, + "pass@16": 0.8402692553045812, + "pass@32": 0.8729885399962004, + "pass@64": 0.8982468595231101, + "pass@128": 0.918918918918919 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 765697, + "answer_accuracy": 0.9325031664068589, + "avg_response_len": 291.1401585639127, + "resp_tokens_sum": 239061007, + "pass_at_k": { + "pass@1": 0.9325031664068589, + "pass@2": 0.9518363817271153, + "pass@4": 0.9646200336464147, + "pass@8": 0.9733466730553039, + "pass@16": 0.9793728174194902, + "pass@32": 0.983754479786111, + "pass@64": 0.9872180069196503, + "pass@128": 0.9900233826968043 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9433713573619632, + "pass@2": 0.9535782540698516, + "pass@4": 0.9618546829460732, + "pass@8": 0.968359742151891, + "pass@16": 0.9741511694787096, + "pass@32": 0.9805862212217205, + "pass@64": 0.9881224160445984, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983901515151515, + "pass@2": 0.999460898353615, + "pass@4": 0.9999225821204167, + "pass@8": 0.9999984049427112, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999813, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935914855072464, + "pass@2": 0.9972134257674314, + "pass@4": 0.9991986126734157, + "pass@8": 0.9998860811105759, + "pass@16": 0.9999969943734291, + "pass@32": 0.9999999992176973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999328438395415, + "pass@2": 0.99999894242267, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981484826589595, + "pass@2": 0.9991423353511447, + "pass@4": 0.9997656553768929, + "pass@8": 0.9999830971478822, + "pass@16": 0.9999999407613452, + "pass@32": 0.9999999999999076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9978213028169014, + "pass@2": 0.998696212709327, + "pass@4": 0.9994122448602376, + "pass@8": 0.9998848700630322, + "pass@16": 0.9999964254002791, + "pass@32": 0.9999999987973975, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9976609281437125, + "pass@2": 0.9988149930454053, + "pass@4": 0.9995776498184733, + "pass@8": 0.9999453661202221, + "pass@16": 0.9999993082440582, + "pass@32": 0.9999999999733661, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9791274472891566, + "pass@2": 0.9844943257281095, + "pass@4": 0.9885222114554959, + "pass@8": 0.9915842526776513, + "pass@16": 0.9936573596958077, + "pass@32": 0.9946784629745986, + "pass@64": 0.9954818436151764, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9556720219435737, + "pass@2": 0.9739207939426853, + "pass@4": 0.9861632180381843, + "pass@8": 0.9932283805113136, + "pass@16": 0.9967175800031935, + "pass@32": 0.9986257066354457, + "pass@64": 0.9997926928260641, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9692752100840336, + "pass@2": 0.9822620977524867, + "pass@4": 0.9899195350056031, + "pass@8": 0.9950258375460176, + "pass@16": 0.9984359616586164, + "pass@32": 0.9998247453476256, + "pass@64": 0.9999989636921891, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9544956140350878, + "pass@2": 0.9714041706957683, + "pass@4": 0.9850686652107085, + "pass@8": 0.9941410138686707, + "pass@16": 0.9982549357238066, + "pass@32": 0.9997482243275714, + "pass@64": 0.9999976075223646, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.962405303030303, + "pass@2": 0.9801527827487472, + "pass@4": 0.9897256777561895, + "pass@8": 0.9949554819709671, + "pass@16": 0.9975806792718727, + "pass@32": 0.9989957428151091, + "pass@64": 0.999819329898887, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9157540954415955, + "pass@2": 0.9383107460125177, + "pass@4": 0.9528681665860143, + "pass@8": 0.9618255529076115, + "pass@16": 0.9666002487818389, + "pass@32": 0.9691287175972142, + "pass@64": 0.9713401996282022, + "pass@128": 0.9743589743589743 + }, + "17": { + "pass@1": 0.8866459627329193, + "pass@2": 0.9140307869125057, + "pass@4": 0.9320510835796145, + "pass@8": 0.9470251241563546, + "pass@16": 0.958828933724438, + "pass@32": 0.9669829599693496, + "pass@64": 0.9726542282720683, + "pass@128": 0.9782608695652174 + }, + "18": { + "pass@1": 0.8592830882352941, + "pass@2": 0.9078276980083376, + "pass@4": 0.9323671411477975, + "pass@8": 0.9477004466579106, + "pass@16": 0.9584144161429146, + "pass@32": 0.9654229587125491, + "pass@64": 0.9701825020374101, + "pass@128": 0.9735294117647059 + }, + "19": { + "pass@1": 0.6903366815476191, + "pass@2": 0.7712544232752157, + "pass@4": 0.8257569664841, + "pass@8": 0.8643543757323078, + "pass@16": 0.8929841094671906, + "pass@32": 0.914312749232494, + "pass@64": 0.9313348502509535, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.6061938622754491, + "pass@2": 0.7069592360554483, + "pass@4": 0.779909319718268, + "pass@8": 0.8312956919046384, + "pass@16": 0.8691291374054729, + "pass@32": 0.9006392983887458, + "pass@64": 0.926877074969234, + "pass@128": 0.9461077844311377 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 745158, + "answer_accuracy": 0.930108144272248, + "avg_response_len": 311.89763365753316, + "resp_tokens_sum": 249877413, + "pass_at_k": { + "pass@1": 0.930108144272248, + "pass@2": 0.9506625813159659, + "pass@4": 0.9639841486302464, + "pass@8": 0.9728187372779841, + "pass@16": 0.9789563417603896, + "pass@32": 0.9835196674940303, + "pass@64": 0.9868039392795732, + "pass@128": 0.9892954146029718 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9640391791044776, + "pass@2": 0.9751006287460335, + "pass@4": 0.9840680005484389, + "pass@8": 0.989525217666148, + "pass@16": 0.9922932987785924, + "pass@32": 0.9943696315299635, + "pass@64": 0.9970265954859788, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998857821637427, + "pass@2": 0.9999978415527007, + "pass@4": 0.9999999997259115, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920266544117647, + "pass@2": 0.9942518961324689, + "pass@4": 0.9955414982134585, + "pass@8": 0.9967116901009395, + "pass@16": 0.998155080969405, + "pass@32": 0.9994198334387632, + "pass@64": 0.9999589349376307, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.9999988356640751, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976513053797469, + "pass@2": 0.9984395245689226, + "pass@4": 0.9992420280930073, + "pass@8": 0.9998284242652762, + "pass@16": 0.9999927222555876, + "pass@32": 0.9999999948561573, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9929147897897898, + "pass@2": 0.9970771706746117, + "pass@4": 0.9993902071362702, + "pass@8": 0.9999651367971397, + "pass@16": 0.9999998583663093, + "pass@32": 0.9999999999994196, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9923884646302251, + "pass@2": 0.9939457427146365, + "pass@4": 0.994643188391644, + "pass@8": 0.995451850487162, + "pass@16": 0.9964930745316808, + "pass@32": 0.9974718962737498, + "pass@64": 0.9983913004056533, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9703125, + "pass@2": 0.9826138920134985, + "pass@4": 0.9905110777819439, + "pass@8": 0.9952361673715535, + "pass@16": 0.9982486278795625, + "pass@32": 0.9997469296363813, + "pass@64": 0.9999980432330696, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9693950688073395, + "pass@2": 0.9864906149919334, + "pass@4": 0.9964843422668495, + "pass@8": 0.999587940904666, + "pass@16": 0.9999840639525752, + "pass@32": 0.9999999728189605, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9416232638888888, + "pass@2": 0.9657007934772044, + "pass@4": 0.9807125428765847, + "pass@8": 0.9889479333237797, + "pass@16": 0.9935783281704044, + "pass@32": 0.9965795260843454, + "pass@64": 0.9983950138936485, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9647626112759644, + "pass@2": 0.9797271840463564, + "pass@4": 0.988019856104782, + "pass@8": 0.992716483465879, + "pass@16": 0.9955701659139958, + "pass@32": 0.9973444053304108, + "pass@64": 0.9984967309084293, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9497010030864198, + "pass@2": 0.9660535597112858, + "pass@4": 0.9750617191369597, + "pass@8": 0.9813197913201593, + "pass@16": 0.9856122764931645, + "pass@32": 0.9887173282223192, + "pass@64": 0.991433089630488, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8941970870206489, + "pass@2": 0.9212493177014375, + "pass@4": 0.9393702139555561, + "pass@8": 0.9519523737165061, + "pass@16": 0.9612349386592156, + "pass@32": 0.9691753292432682, + "pass@64": 0.9747878703756769, + "pass@128": 0.976401179941003 + }, + "17": { + "pass@1": 0.8594455948795181, + "pass@2": 0.897720433900958, + "pass@4": 0.922174053073938, + "pass@8": 0.9382293341864976, + "pass@16": 0.9493378639982005, + "pass@32": 0.959124887388815, + "pass@64": 0.9681289020289342, + "pass@128": 0.9759036144578314 + }, + "18": { + "pass@1": 0.8485887096774194, + "pass@2": 0.903131746888494, + "pass@4": 0.9370043603420539, + "pass@8": 0.9578532946585311, + "pass@16": 0.9702169368171149, + "pass@32": 0.9777537348282815, + "pass@64": 0.9826360654372754, + "pass@128": 0.9870967741935484 + }, + "19": { + "pass@1": 0.7076331193353474, + "pass@2": 0.7887924625924787, + "pass@4": 0.8411277290300948, + "pass@8": 0.878659443062788, + "pass@16": 0.9053831900865986, + "pass@32": 0.9237568606242907, + "pass@64": 0.9352225250892545, + "pass@128": 0.9425981873111783 + }, + "20": { + "pass@1": 0.626196509009009, + "pass@2": 0.7125315522215124, + "pass@4": 0.7734252413268156, + "pass@8": 0.8190036582565382, + "pass@16": 0.8555370910100033, + "pass@32": 0.8847167374444287, + "pass@64": 0.9059201413232556, + "pass@128": 0.9219219219219219 + } + } + } + } + } +] \ No newline at end of file diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl b/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ad7400a6cf8ceb619e7beb2ce81c9bb6f6659caf --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bc96c6c038c1ffd5920d3659d33c7102aa384e31647ed37d10c1b8001dd287d +size 1789699126 diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_metrics.json b/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0d86728cfbe280c3d00b147881ec97820b930b05 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-38645", + "id": { + "answer_accuracy": 0.9926675347222222, + "avg_response_len": 200.20069097222222, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9926675347222222, + "pass@2": 0.9947181075021866, + "pass@4": 0.9961613556430446, + "pass@8": 0.9973026695761072, + "pass@16": 0.9981802787310782, + "pass@32": 0.9987948199302888, + "pass@64": 0.9992436062012987, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9546875, + "pass@2": 0.9643489173228345, + "pass@4": 0.9724239796587925, + "pass@8": 0.9796451423626065, + "pass@16": 0.9858481869460654, + "pass@32": 0.9905653033141515, + "pass@64": 0.9942351004838736, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99915625, + "pass@2": 0.9997679625984253, + "pass@4": 0.9999731251406074, + "pass@8": 0.9999995472606525, + "pass@16": 0.9999999999231317, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99971875, + "pass@2": 0.9999778543307086, + "pass@4": 0.9999998386764155, + "pass@8": 0.9999999999969994, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941953125, + "pass@2": 0.9956927903543309, + "pass@4": 0.9968433910761155, + "pass@8": 0.9977670040034647, + "pass@16": 0.9983917733914977, + "pass@32": 0.9987650573014524, + "pass@64": 0.9989711801510246, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99975, + "pass@2": 0.9999803149606299, + "pass@4": 0.9999997736220472, + "pass@8": 0.9999999999829965, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958359375, + "pass@2": 0.9972054625984251, + "pass@4": 0.9982031517622797, + "pass@8": 0.9990104842344203, + "pass@16": 0.9995193774344288, + "pass@32": 0.9998284427302075, + "pass@64": 0.9999861762904964, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9950703125, + "pass@2": 0.997607160433071, + "pass@4": 0.9989324190101235, + "pass@8": 0.9996309201327338, + "pass@16": 0.9999226231371163, + "pass@32": 0.9999963916027095, + "pass@64": 0.9999999990502599, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.99559375, + "pass@2": 0.9978825049212597, + "pass@4": 0.9990765218410199, + "pass@8": 0.9996709282110962, + "pass@16": 0.9999405477474613, + "pass@32": 0.9999981844240781, + "pass@64": 0.9999999998360332, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9546875, + "2": 1.0, + "3": 0.99915625, + "4": 0.99971875, + "5": 0.9941953125, + "6": 0.99975, + "7": 0.9958359375, + "8": 0.9950703125, + "9": 0.99559375 + }, + "per_op_avg_response_len": { + "10": 281.6347734375, + "2": 115.6326015625, + "3": 144.829234375, + "4": 159.923921875, + "5": 173.2866484375, + "6": 198.9441953125, + "7": 224.1663515625, + "8": 243.7276171875, + "9": 259.660875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.6347734375, + "2": 115.6326015625, + "3": 144.829234375, + "4": 159.923921875, + "5": 173.2866484375, + "6": 198.9441953125, + "7": 224.1663515625, + "8": 243.7276171875, + "9": 259.660875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376612, + "answer_accuracy": 0.9936782337048294, + "avg_response_len": 190.71223826409997, + "resp_tokens_sum": 72281464, + "pass_at_k": { + "pass@1": 0.9936782337048294, + "pass@2": 0.9956697567059437, + "pass@4": 0.996967571777692, + "pass@8": 0.9979938409459532, + "pass@16": 0.9988097707718845, + "pass@32": 0.9992490341086927, + "pass@64": 0.9994945000973269, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9615366887905604, + "pass@2": 0.970145808654449, + "pass@4": 0.9771259859110534, + "pass@8": 0.9837142589656627, + "pass@16": 0.9897806295670789, + "pass@32": 0.9934434017257641, + "pass@64": 0.9955847044260153, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997610856269113, + "pass@2": 0.9999864552481399, + "pass@4": 0.9999999638806617, + "pass@8": 0.9999999999999809, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99931640625, + "pass@2": 0.9999350239911416, + "pass@4": 0.9999994973284589, + "pass@8": 0.999999999990623, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9972468285214349, + "pass@4": 0.9985486585010207, + "pass@8": 0.9994671341886722, + "pass@16": 0.9999236250321564, + "pass@32": 0.9999990493973, + "pass@64": 0.9999999999949286, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995555763473054, + "pass@2": 0.9999499033429206, + "pass@4": 0.9999993320445721, + "pass@8": 0.9999999999490911, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964404585798816, + "pass@2": 0.9978735207100592, + "pass@4": 0.9986706386183974, + "pass@8": 0.9994230191252826, + "pass@16": 0.9999007629980622, + "pass@32": 0.9999981593967747, + "pass@64": 0.99999999996853, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.997245592948718, + "pass@2": 0.9984009848828992, + "pass@4": 0.9992322474113812, + "pass@8": 0.9998262245762686, + "pass@16": 0.9999926289511722, + "pass@32": 0.9999999947902106, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950704225352113, + "pass@2": 0.9983818759010756, + "pass@4": 0.999779442534472, + "pass@8": 0.9999935759719568, + "pass@16": 0.9999999938621692, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387758, + "answer_accuracy": 0.9925817087155964, + "avg_response_len": 198.13595081094365, + "resp_tokens_sum": 77402998, + "pass_at_k": { + "pass@1": 0.9925817087155964, + "pass@2": 0.9942163103321948, + "pass@4": 0.9953949149315869, + "pass@8": 0.9964613581820646, + "pass@16": 0.9974663333818123, + "pass@32": 0.9984046891944619, + "pass@64": 0.999174331646805, + "pass@128": 0.9996723460026212 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9446654524539877, + "pass@2": 0.9550259498333415, + "pass@4": 0.9634138444542589, + "pass@8": 0.970866877957459, + "pass@16": 0.9782378206275941, + "pass@32": 0.9857900637806933, + "pass@64": 0.9923585282128631, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988873106060606, + "pass@2": 0.9997856269386781, + "pass@4": 0.9999895822681255, + "pass@8": 0.9999999777033803, + "pass@16": 0.9999999999999803, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9965579710144927, + "pass@2": 0.9972633515919205, + "pass@4": 0.9975327839454851, + "pass@8": 0.9979120550709301, + "pass@16": 0.9985302315822023, + "pass@32": 0.9993303337082865, + "pass@64": 0.9999164669633687, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999915393813595, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9975162572254336, + "pass@2": 0.9985822931364069, + "pass@4": 0.9993384239181086, + "pass@8": 0.9998572295938166, + "pass@16": 0.9999945349140869, + "pass@32": 0.9999999969725001, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9978297937229678, + "pass@4": 0.9984739589769589, + "pass@8": 0.9991927685739189, + "pass@16": 0.9997893254704586, + "pass@32": 0.999989840105361, + "pass@64": 0.9999999973246758, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9973100673652695, + "pass@2": 0.9978701553585743, + "pass@4": 0.9984959510113631, + "pass@8": 0.9992628054872507, + "pass@16": 0.9998367378399854, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379183, + "answer_accuracy": 0.9917533269166388, + "avg_response_len": 211.7162234265149, + "resp_tokens_sum": 80946734, + "pass_at_k": { + "pass@1": 0.9917533269166388, + "pass@2": 0.9942874585803578, + "pass@4": 0.996145276324892, + "pass@8": 0.997477133569582, + "pass@16": 0.9982857475202078, + "pass@32": 0.9987431797640651, + "pass@64": 0.9990656782180966, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9575093283582089, + "pass@2": 0.9675553090844993, + "pass@4": 0.9764339019189767, + "pass@8": 0.9840698697287039, + "pass@16": 0.9892746985021801, + "pass@32": 0.9922997890645133, + "pass@64": 0.9946955387643621, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988383746355685, + "pass@2": 0.9995426671793577, + "pass@4": 0.9999317049654508, + "pass@8": 0.999998701511788, + "pass@16": 0.9999999997759129, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998629385964912, + "pass@2": 0.9999964025878345, + "pass@4": 0.9999999986295574, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906020220588235, + "pass@2": 0.9926593619731356, + "pass@4": 0.9945639799620636, + "pass@8": 0.9960446992193006, + "pass@16": 0.9968320635603195, + "pass@32": 0.997048210623365, + "pass@64": 0.9970588207360587, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9999753548895899, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9933494857594937, + "pass@2": 0.9949833518140139, + "pass@4": 0.9964600671949234, + "pass@8": 0.9976420959829561, + "pass@16": 0.9985911723443348, + "pass@32": 0.9994590693089006, + "pass@64": 0.9999562541175095, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913194444444444, + "pass@2": 0.9966260551890473, + "pass@4": 0.9991402473902474, + "pass@8": 0.9999150306942846, + "pass@16": 0.9999991362232366, + "pass@32": 0.9999999999722541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943478697749196, + "pass@2": 0.9973257462592097, + "pass@4": 0.9988976594967428, + "pass@8": 0.9997409379688414, + "pass@16": 0.9999841784174797, + "pass@32": 0.999999971428724, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.844671875, + "avg_response_len": 377.0967484375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.844671875, + "pass@2": 0.8858922244094257, + "pass@4": 0.9141995808398893, + "pass@8": 0.9341837080333515, + "pass@16": 0.9486755484175874, + "pass@32": 0.9595485287445626, + "pass@64": 0.9677227239017148, + "pass@128": 0.9738 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9726171875, + "pass@2": 0.9828142224409449, + "pass@4": 0.9897272834645668, + "pass@8": 0.994115282605061, + "pass@16": 0.9967868462201568, + "pass@32": 0.9981416746464655, + "pass@64": 0.9987367902594374, + "pass@128": 0.999 + }, + "12": { + "pass@1": 0.9644375, + "pass@2": 0.9804958169291337, + "pass@4": 0.9905623012748407, + "pass@8": 0.9955742374047644, + "pass@16": 0.9976946487441701, + "pass@32": 0.998880341662968, + "pass@64": 0.9996771390867283, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.94846875, + "pass@2": 0.9692981053149601, + "pass@4": 0.9827727335958009, + "pass@8": 0.9904335216979527, + "pass@16": 0.9947883099927657, + "pass@32": 0.9972493649171777, + "pass@64": 0.9987378705005298, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9449453125, + "pass@2": 0.964385088582677, + "pass@4": 0.9777694221034873, + "pass@8": 0.9865351007090998, + "pass@16": 0.9922419706883057, + "pass@32": 0.9962460277292379, + "pass@64": 0.9986672351003253, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.952953125, + "pass@2": 0.9735356791338584, + "pass@4": 0.9843826020809897, + "pass@8": 0.9903981612161823, + "pass@16": 0.9936567666222805, + "pass@32": 0.9955643444237633, + "pass@64": 0.9966508344175822, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.8934609375, + "pass@2": 0.9216697834645677, + "pass@4": 0.9393977767154105, + "pass@8": 0.9517931687352534, + "pass@16": 0.96139777462629, + "pass@32": 0.9693210349487325, + "pass@64": 0.9751328706018372, + "pass@128": 0.979 + }, + "17": { + "pass@1": 0.8661875, + "pass@2": 0.9038288631889774, + "pass@4": 0.9277408284589423, + "pass@8": 0.9448044024114316, + "pass@16": 0.9568565852772714, + "pass@32": 0.9646994651551266, + "pass@64": 0.9694077197479471, + "pass@128": 0.972 + }, + "18": { + "pass@1": 0.788859375, + "pass@2": 0.8561178641732294, + "pass@4": 0.8988476714473197, + "pass@8": 0.9267583655390567, + "pass@16": 0.9448361294806787, + "pass@32": 0.9568568515114735, + "pass@64": 0.9657552890983766, + "pass@128": 0.973 + }, + "19": { + "pass@1": 0.625640625, + "pass@2": 0.7169927411417328, + "pass@4": 0.7811060572740908, + "pass@8": 0.8285917496219818, + "pass@16": 0.8654698909087529, + "pass@32": 0.8941006950628526, + "pass@64": 0.9155477279248722, + "pass@128": 0.932 + }, + "20": { + "pass@1": 0.4891484375, + "pass@2": 0.5897840797244098, + "pass@4": 0.6696891319835014, + "pass@8": 0.7328330903927104, + "pass@16": 0.7830265616151493, + "pass@32": 0.824425487387794, + "pass@64": 0.8589137622794993, + "pass@128": 0.886 + } + }, + "per_op_accuracy": { + "11": 0.9726171875, + "12": 0.9644375, + "13": 0.94846875, + "14": 0.9449453125, + "15": 0.952953125, + "16": 0.8934609375, + "17": 0.8661875, + "18": 0.788859375, + "19": 0.625640625, + "20": 0.4891484375 + }, + "per_op_avg_response_len": { + "11": 300.8103125, + "12": 313.388375, + "13": 334.3829921875, + "14": 361.2994765625, + "15": 381.435625, + "16": 390.2542265625, + "17": 410.935984375, + "18": 425.279296875, + "19": 422.265296875, + "20": 430.9158984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 300.8103125, + "12": 313.388375, + "13": 334.3829921875, + "14": 361.2994765625, + "15": 381.435625, + "16": 390.2542265625, + "17": 410.935984375, + "18": 425.279296875, + "19": 422.265296875, + "20": 430.9158984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 361406, + "answer_accuracy": 0.8390741084695393, + "avg_response_len": 358.89673569836555, + "resp_tokens_sum": 154584002, + "pass_at_k": { + "pass@1": 0.8390741084695393, + "pass@2": 0.8810461662435247, + "pass@4": 0.910567308794798, + "pass@8": 0.9312514898839028, + "pass@16": 0.9461738981519632, + "pass@32": 0.9574585135295627, + "pass@64": 0.9658631665868088, + "pass@128": 0.9723625557206538 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9698123229461756, + "pass@2": 0.981584311079387, + "pass@4": 0.9891040887948495, + "pass@8": 0.9937797636155625, + "pass@16": 0.9970789483598115, + "pass@32": 0.999223562425918, + "pass@64": 0.9999570534856826, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9659030720338984, + "pass@2": 0.9809874632990792, + "pass@4": 0.9906215364498931, + "pass@8": 0.9956137072056146, + "pass@16": 0.9978803736651605, + "pass@32": 0.9993246177441057, + "pass@64": 0.9999568688255516, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9498922413793104, + "pass@2": 0.9708758608347934, + "pass@4": 0.9840897281060871, + "pass@8": 0.9909108589255634, + "pass@16": 0.9943999521503089, + "pass@32": 0.9963005415376021, + "pass@64": 0.9980499904733999, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9233839563862928, + "pass@2": 0.944520460789364, + "pass@4": 0.9611923819102052, + "pass@8": 0.9751079976868544, + "pass@16": 0.9861293435926446, + "pass@32": 0.9935272719026856, + "pass@64": 0.9975030863628863, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9595375722543352, + "pass@2": 0.9784015719584908, + "pass@4": 0.987961396353635, + "pass@8": 0.9923778119086278, + "pass@16": 0.9938310119401556, + "pass@32": 0.9941908530213401, + "pass@64": 0.9942196182961773, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8793850806451613, + "pass@2": 0.9133687611125221, + "pass@4": 0.9350571322737883, + "pass@8": 0.9489252481008612, + "pass@16": 0.9594660322806725, + "pass@32": 0.9677192403291304, + "pass@64": 0.9722136540329196, + "pass@128": 0.9741935483870968 + }, + "17": { + "pass@1": 0.8896992413294798, + "pass@2": 0.9255197191752769, + "pass@4": 0.9464512051600483, + "pass@8": 0.9595738090472379, + "pass@16": 0.9673121876199731, + "pass@32": 0.9725731938539892, + "pass@64": 0.9768482847119901, + "pass@128": 0.9797687861271677 + }, + "18": { + "pass@1": 0.7444419642857143, + "pass@2": 0.8136916479190104, + "pass@4": 0.8625618206652741, + "pass@8": 0.8958082898451949, + "pass@16": 0.9174575405950798, + "pass@32": 0.9337781303810576, + "pass@64": 0.947397000627292, + "pass@128": 0.9571428571428572 + }, + "19": { + "pass@1": 0.6253753753753754, + "pass@2": 0.7184758577475104, + "pass@4": 0.7833039738275956, + "pass@8": 0.8296772377534075, + "pass@16": 0.8660549147719705, + "pass@32": 0.8940814351693145, + "pass@64": 0.9133406551952218, + "pass@128": 0.9279279279279279 + }, + "20": { + "pass@1": 0.4760932807807808, + "pass@2": 0.5768932071835613, + "pass@4": 0.6601997517765237, + "pass@8": 0.7265525943478015, + "pass@16": 0.7789762742387952, + "pass@32": 0.8215439955324653, + "pass@64": 0.8572865791166149, + "pass@128": 0.8888888888888888 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 365670, + "answer_accuracy": 0.8494787020517395, + "avg_response_len": 372.46568586455544, + "resp_tokens_sum": 160333069, + "pass_at_k": { + "pass@1": 0.8494787020517395, + "pass@2": 0.8895784091467879, + "pass@4": 0.9172445349438386, + "pass@8": 0.9367987196255487, + "pass@16": 0.950993996091102, + "pass@32": 0.961825739478423, + "pass@64": 0.9700731776529568, + "pass@128": 0.976211715730003 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9770566641566265, + "pass@2": 0.9841648829570252, + "pass@4": 0.989164249235412, + "pass@8": 0.9922976666641117, + "pass@16": 0.9941890352635745, + "pass@32": 0.9952655846372421, + "pass@64": 0.9962408325530068, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9620640673981191, + "pass@2": 0.9773606620097254, + "pass@4": 0.9866140545127782, + "pass@8": 0.9920244775588233, + "pass@16": 0.9951791105256566, + "pass@32": 0.9972397387226369, + "pass@64": 0.9990357602586, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9582020308123249, + "pass@2": 0.9730195720020293, + "pass@4": 0.9829933805543213, + "pass@8": 0.9895737184859488, + "pass@16": 0.9944316023575817, + "pass@32": 0.9979028151511814, + "pass@64": 0.9996096734768417, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9545869883040936, + "pass@2": 0.9759829569001243, + "pass@4": 0.9895990122945156, + "pass@8": 0.9958761100329401, + "pass@16": 0.9982192698442767, + "pass@32": 0.9995253069447078, + "pass@64": 0.9999798903618302, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.964654356060606, + "pass@2": 0.9848753280839894, + "pass@4": 0.994307931394939, + "pass@8": 0.9984355730943542, + "pass@16": 0.9997865737709953, + "pass@32": 0.999996225078149, + "pass@64": 0.9999999999355342, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9025106837606838, + "pass@2": 0.9249840865244409, + "pass@4": 0.9394740470475377, + "pass@8": 0.951603221594124, + "pass@16": 0.9618929894371758, + "pass@32": 0.9704062836352245, + "pass@64": 0.9772483460299233, + "pass@128": 0.9829059829059829 + }, + "17": { + "pass@1": 0.8692498059006211, + "pass@2": 0.9090858377757125, + "pass@4": 0.9353135909447651, + "pass@8": 0.9535516185363221, + "pass@16": 0.965466357509199, + "pass@32": 0.9717789256494813, + "pass@64": 0.9742932379404277, + "pass@128": 0.9751552795031055 + }, + "18": { + "pass@1": 0.8068244485294118, + "pass@2": 0.8750351001621121, + "pass@4": 0.9166487397714989, + "pass@8": 0.9411302798270424, + "pass@16": 0.9559175964520414, + "pass@32": 0.9659157855028336, + "pass@64": 0.9733500817730232, + "pass@128": 0.9794117647058823 + }, + "19": { + "pass@1": 0.6194428943452381, + "pass@2": 0.7078185051087363, + "pass@4": 0.7742094414537468, + "pass@8": 0.8272903079106774, + "pass@16": 0.8697325655416732, + "pass@32": 0.9032315125581434, + "pass@64": 0.9280735994123079, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.47754491017964074, + "pass@2": 0.5815400449101793, + "pass@4": 0.6627141611041132, + "pass@8": 0.7251319869091699, + "pass@16": 0.7741932335037112, + "pass@32": 0.8160539579293532, + "pass@64": 0.8519449321090409, + "pass@128": 0.8802395209580839 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 354104, + "answer_accuracy": 0.8454882334963325, + "avg_response_len": 400.5739202895782, + "resp_tokens_sum": 167766767, + "pass_at_k": { + "pass@1": 0.8454882334963325, + "pass@2": 0.8870873180698133, + "pass@4": 0.914805453327734, + "pass@8": 0.93451152896499, + "pass@16": 0.9488653753178875, + "pass@32": 0.9593574045087645, + "pass@64": 0.967219311432032, + "pass@128": 0.972799511002445 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9710813492063493, + "pass@2": 0.9827689507561553, + "pass@4": 0.9910190773772323, + "pass@8": 0.9964069864008952, + "pass@16": 0.9991975166401181, + "pass@32": 0.9999605809858803, + "pass@64": 0.9999999808615654, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9651662844036697, + "pass@2": 0.9830220297864142, + "pass@4": 0.9943498286911382, + "pass@8": 0.9989944241978972, + "pass@16": 0.9999475847370622, + "pass@32": 0.9999998481010195, + "pass@64": 0.9999999999999678, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9363425925925926, + "pass@2": 0.9636441929133857, + "pass@4": 0.9812329428150182, + "pass@8": 0.9909109265469578, + "pass@16": 0.995563713627039, + "pass@32": 0.997463540610219, + "pass@64": 0.9984545373712423, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9556982566765578, + "pass@2": 0.971536645131896, + "pass@4": 0.9815543184142042, + "pass@8": 0.9879400706835424, + "pass@16": 0.9919984009149101, + "pass@32": 0.9955077699507003, + "pass@64": 0.9984439818222332, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9340036651234568, + "pass@2": 0.9567897437299503, + "pass@4": 0.9704516715503154, + "pass@8": 0.9800978369588287, + "pass@16": 0.9872273677362894, + "pass@32": 0.9925171450697239, + "pass@64": 0.9958359460135142, + "pass@128": 0.9969135802469136 + }, + "16": { + "pass@1": 0.8969625737463127, + "pass@2": 0.9258290653380717, + "pass@4": 0.9432881274243372, + "pass@8": 0.9546124219600274, + "pass@16": 0.9626515201381516, + "pass@32": 0.9696621383207606, + "pass@64": 0.9756120011655725, + "pass@128": 0.9793510324483776 + }, + "17": { + "pass@1": 0.8387142319277109, + "pass@2": 0.8761247005739499, + "pass@4": 0.9008967927390097, + "pass@8": 0.9209284437120231, + "pass@16": 0.937609642297589, + "pass@32": 0.9496274880196173, + "pass@64": 0.9569150319903035, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.819304435483871, + "pass@2": 0.8832704946659896, + "pass@4": 0.9202918151360103, + "pass@8": 0.9459392546840153, + "pass@16": 0.9635935725119584, + "pass@32": 0.9729778671198072, + "pass@64": 0.9781526163741832, + "pass@128": 0.9838709677419355 + }, + "19": { + "pass@1": 0.6321988293051359, + "pass@2": 0.7248134827651832, + "pass@4": 0.7858956545650826, + "pass@8": 0.828820803601541, + "pass@16": 0.8605542666697465, + "pass@32": 0.8848513260783547, + "pass@64": 0.905053052393738, + "pass@128": 0.9214501510574018 + }, + "20": { + "pass@1": 0.5138419669669669, + "pass@2": 0.6109437439407909, + "pass@4": 0.6861744289283656, + "pass@8": 0.7468378163580444, + "pass@16": 0.7959367036137914, + "pass@32": 0.8357036484296656, + "pass@64": 0.8675307030307697, + "pass@128": 0.8888888888888888 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9147750822368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9147750822368421, + "pass@2": 0.9374413269270534, + "pass@4": 0.9530235794308524, + "pass@8": 0.9640821635009731, + "pass@16": 0.9721251575134345, + "pass@32": 0.9781388772009632, + "pass@64": 0.9826536681488868, + "pass@128": 0.986 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9546875, + "pass@2": 0.9643489173228345, + "pass@4": 0.9724239796587925, + "pass@8": 0.9796451423626065, + "pass@16": 0.9858481869460654, + "pass@32": 0.9905653033141515, + "pass@64": 0.9942351004838736, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99915625, + "pass@2": 0.9997679625984253, + "pass@4": 0.9999731251406074, + "pass@8": 0.9999995472606525, + "pass@16": 0.9999999999231317, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99971875, + "pass@2": 0.9999778543307086, + "pass@4": 0.9999998386764155, + "pass@8": 0.9999999999969994, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941953125, + "pass@2": 0.9956927903543309, + "pass@4": 0.9968433910761155, + "pass@8": 0.9977670040034647, + "pass@16": 0.9983917733914977, + "pass@32": 0.9987650573014524, + "pass@64": 0.9989711801510246, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99975, + "pass@2": 0.9999803149606299, + "pass@4": 0.9999997736220472, + "pass@8": 0.9999999999829965, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958359375, + "pass@2": 0.9972054625984251, + "pass@4": 0.9982031517622797, + "pass@8": 0.9990104842344203, + "pass@16": 0.9995193774344288, + "pass@32": 0.9998284427302075, + "pass@64": 0.9999861762904964, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9950703125, + "pass@2": 0.997607160433071, + "pass@4": 0.9989324190101235, + "pass@8": 0.9996309201327338, + "pass@16": 0.9999226231371163, + "pass@32": 0.9999963916027095, + "pass@64": 0.9999999990502599, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.99559375, + "pass@2": 0.9978825049212597, + "pass@4": 0.9990765218410199, + "pass@8": 0.9996709282110962, + "pass@16": 0.9999405477474613, + "pass@32": 0.9999981844240781, + "pass@64": 0.9999999998360332, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9726171875, + "pass@2": 0.9828142224409449, + "pass@4": 0.9897272834645668, + "pass@8": 0.994115282605061, + "pass@16": 0.9967868462201568, + "pass@32": 0.9981416746464655, + "pass@64": 0.9987367902594374, + "pass@128": 0.999 + }, + "12": { + "pass@1": 0.9644375, + "pass@2": 0.9804958169291337, + "pass@4": 0.9905623012748407, + "pass@8": 0.9955742374047644, + "pass@16": 0.9976946487441701, + "pass@32": 0.998880341662968, + "pass@64": 0.9996771390867283, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.94846875, + "pass@2": 0.9692981053149601, + "pass@4": 0.9827727335958009, + "pass@8": 0.9904335216979527, + "pass@16": 0.9947883099927657, + "pass@32": 0.9972493649171777, + "pass@64": 0.9987378705005298, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9449453125, + "pass@2": 0.964385088582677, + "pass@4": 0.9777694221034873, + "pass@8": 0.9865351007090998, + "pass@16": 0.9922419706883057, + "pass@32": 0.9962460277292379, + "pass@64": 0.9986672351003253, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.952953125, + "pass@2": 0.9735356791338584, + "pass@4": 0.9843826020809897, + "pass@8": 0.9903981612161823, + "pass@16": 0.9936567666222805, + "pass@32": 0.9955643444237633, + "pass@64": 0.9966508344175822, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.8934609375, + "pass@2": 0.9216697834645677, + "pass@4": 0.9393977767154105, + "pass@8": 0.9517931687352534, + "pass@16": 0.96139777462629, + "pass@32": 0.9693210349487325, + "pass@64": 0.9751328706018372, + "pass@128": 0.979 + }, + "17": { + "pass@1": 0.8661875, + "pass@2": 0.9038288631889774, + "pass@4": 0.9277408284589423, + "pass@8": 0.9448044024114316, + "pass@16": 0.9568565852772714, + "pass@32": 0.9646994651551266, + "pass@64": 0.9694077197479471, + "pass@128": 0.972 + }, + "18": { + "pass@1": 0.788859375, + "pass@2": 0.8561178641732294, + "pass@4": 0.8988476714473197, + "pass@8": 0.9267583655390567, + "pass@16": 0.9448361294806787, + "pass@32": 0.9568568515114735, + "pass@64": 0.9657552890983766, + "pass@128": 0.973 + }, + "19": { + "pass@1": 0.625640625, + "pass@2": 0.7169927411417328, + "pass@4": 0.7811060572740908, + "pass@8": 0.8285917496219818, + "pass@16": 0.8654698909087529, + "pass@32": 0.8941006950628526, + "pass@64": 0.9155477279248722, + "pass@128": 0.932 + }, + "20": { + "pass@1": 0.4891484375, + "pass@2": 0.5897840797244098, + "pass@4": 0.6696891319835014, + "pass@8": 0.7328330903927104, + "pass@16": 0.7830265616151493, + "pass@32": 0.824425487387794, + "pass@64": 0.8589137622794993, + "pass@128": 0.886 + } + }, + "per_op_accuracy": { + "10": 0.9546875, + "2": 1.0, + "3": 0.99915625, + "4": 0.99971875, + "5": 0.9941953125, + "6": 0.99975, + "7": 0.9958359375, + "8": 0.9950703125, + "9": 0.99559375, + "11": 0.9726171875, + "12": 0.9644375, + "13": 0.94846875, + "14": 0.9449453125, + "15": 0.952953125, + "16": 0.8934609375, + "17": 0.8661875, + "18": 0.788859375, + "19": 0.625640625, + "20": 0.4891484375 + }, + "per_op_avg_response_len": { + "10": 281.6347734375, + "2": 115.6326015625, + "3": 144.829234375, + "4": 159.923921875, + "5": 173.2866484375, + "6": 198.9441953125, + "7": 224.1663515625, + "8": 243.7276171875, + "9": 259.660875, + "11": 300.8103125, + "12": 313.388375, + "13": 334.3829921875, + "14": 361.2994765625, + "15": 381.435625, + "16": 390.2542265625, + "17": 410.935984375, + "18": 425.279296875, + "19": 422.265296875, + "20": 430.9158984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.6347734375, + "2": 115.6326015625, + "3": 144.829234375, + "4": 159.923921875, + "5": 173.2866484375, + "6": 198.9441953125, + "7": 224.1663515625, + "8": 243.7276171875, + "9": 259.660875, + "11": 300.8103125, + "12": 313.388375, + "13": 334.3829921875, + "14": 361.2994765625, + "15": 381.435625, + "16": 390.2542265625, + "17": 410.935984375, + "18": 425.279296875, + "19": 422.265296875, + "20": 430.9158984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 738018, + "answer_accuracy": 0.9114393969332911, + "avg_response_len": 280.1749056473285, + "resp_tokens_sum": 226865466, + "pass_at_k": { + "pass@1": 0.9114393969332911, + "pass@2": 0.9346978341788974, + "pass@4": 0.9510085321100552, + "pass@8": 0.9624914679892927, + "pass@16": 0.9708110810206952, + "pass@32": 0.9770193310184669, + "pass@64": 0.981604927339997, + "pass@128": 0.9851406892190958 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9615366887905604, + "pass@2": 0.970145808654449, + "pass@4": 0.9771259859110534, + "pass@8": 0.9837142589656627, + "pass@16": 0.9897806295670789, + "pass@32": 0.9934434017257641, + "pass@64": 0.9955847044260153, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997610856269113, + "pass@2": 0.9999864552481399, + "pass@4": 0.9999999638806617, + "pass@8": 0.9999999999999809, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99931640625, + "pass@2": 0.9999350239911416, + "pass@4": 0.9999994973284589, + "pass@8": 0.999999999990623, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9972468285214349, + "pass@4": 0.9985486585010207, + "pass@8": 0.9994671341886722, + "pass@16": 0.9999236250321564, + "pass@32": 0.9999990493973, + "pass@64": 0.9999999999949286, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995555763473054, + "pass@2": 0.9999499033429206, + "pass@4": 0.9999993320445721, + "pass@8": 0.9999999999490911, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964404585798816, + "pass@2": 0.9978735207100592, + "pass@4": 0.9986706386183974, + "pass@8": 0.9994230191252826, + "pass@16": 0.9999007629980622, + "pass@32": 0.9999981593967747, + "pass@64": 0.99999999996853, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.997245592948718, + "pass@2": 0.9984009848828992, + "pass@4": 0.9992322474113812, + "pass@8": 0.9998262245762686, + "pass@16": 0.9999926289511722, + "pass@32": 0.9999999947902106, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950704225352113, + "pass@2": 0.9983818759010756, + "pass@4": 0.999779442534472, + "pass@8": 0.9999935759719568, + "pass@16": 0.9999999938621692, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9698123229461756, + "pass@2": 0.981584311079387, + "pass@4": 0.9891040887948495, + "pass@8": 0.9937797636155625, + "pass@16": 0.9970789483598115, + "pass@32": 0.999223562425918, + "pass@64": 0.9999570534856826, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9659030720338984, + "pass@2": 0.9809874632990792, + "pass@4": 0.9906215364498931, + "pass@8": 0.9956137072056146, + "pass@16": 0.9978803736651605, + "pass@32": 0.9993246177441057, + "pass@64": 0.9999568688255516, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9498922413793104, + "pass@2": 0.9708758608347934, + "pass@4": 0.9840897281060871, + "pass@8": 0.9909108589255634, + "pass@16": 0.9943999521503089, + "pass@32": 0.9963005415376021, + "pass@64": 0.9980499904733999, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9233839563862928, + "pass@2": 0.944520460789364, + "pass@4": 0.9611923819102052, + "pass@8": 0.9751079976868544, + "pass@16": 0.9861293435926446, + "pass@32": 0.9935272719026856, + "pass@64": 0.9975030863628863, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9595375722543352, + "pass@2": 0.9784015719584908, + "pass@4": 0.987961396353635, + "pass@8": 0.9923778119086278, + "pass@16": 0.9938310119401556, + "pass@32": 0.9941908530213401, + "pass@64": 0.9942196182961773, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8793850806451613, + "pass@2": 0.9133687611125221, + "pass@4": 0.9350571322737883, + "pass@8": 0.9489252481008612, + "pass@16": 0.9594660322806725, + "pass@32": 0.9677192403291304, + "pass@64": 0.9722136540329196, + "pass@128": 0.9741935483870968 + }, + "17": { + "pass@1": 0.8896992413294798, + "pass@2": 0.9255197191752769, + "pass@4": 0.9464512051600483, + "pass@8": 0.9595738090472379, + "pass@16": 0.9673121876199731, + "pass@32": 0.9725731938539892, + "pass@64": 0.9768482847119901, + "pass@128": 0.9797687861271677 + }, + "18": { + "pass@1": 0.7444419642857143, + "pass@2": 0.8136916479190104, + "pass@4": 0.8625618206652741, + "pass@8": 0.8958082898451949, + "pass@16": 0.9174575405950798, + "pass@32": 0.9337781303810576, + "pass@64": 0.947397000627292, + "pass@128": 0.9571428571428572 + }, + "19": { + "pass@1": 0.6253753753753754, + "pass@2": 0.7184758577475104, + "pass@4": 0.7833039738275956, + "pass@8": 0.8296772377534075, + "pass@16": 0.8660549147719705, + "pass@32": 0.8940814351693145, + "pass@64": 0.9133406551952218, + "pass@128": 0.9279279279279279 + }, + "20": { + "pass@1": 0.4760932807807808, + "pass@2": 0.5768932071835613, + "pass@4": 0.6601997517765237, + "pass@8": 0.7265525943478015, + "pass@16": 0.7789762742387952, + "pass@32": 0.8215439955324653, + "pass@64": 0.8572865791166149, + "pass@128": 0.8888888888888888 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 753428, + "answer_accuracy": 0.9175613795791114, + "avg_response_len": 289.52658198558066, + "resp_tokens_sum": 237736067, + "pass_at_k": { + "pass@1": 0.9175613795791114, + "pass@2": 0.9393609304901669, + "pass@4": 0.9544253548538294, + "pass@8": 0.9651838128249977, + "pass@16": 0.9731036723828017, + "pass@32": 0.9792285383144879, + "pass@64": 0.9839183408625037, + "pass@128": 0.987373343725643 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9446654524539877, + "pass@2": 0.9550259498333415, + "pass@4": 0.9634138444542589, + "pass@8": 0.970866877957459, + "pass@16": 0.9782378206275941, + "pass@32": 0.9857900637806933, + "pass@64": 0.9923585282128631, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988873106060606, + "pass@2": 0.9997856269386781, + "pass@4": 0.9999895822681255, + "pass@8": 0.9999999777033803, + "pass@16": 0.9999999999999803, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9965579710144927, + "pass@2": 0.9972633515919205, + "pass@4": 0.9975327839454851, + "pass@8": 0.9979120550709301, + "pass@16": 0.9985302315822023, + "pass@32": 0.9993303337082865, + "pass@64": 0.9999164669633687, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999915393813595, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9975162572254336, + "pass@2": 0.9985822931364069, + "pass@4": 0.9993384239181086, + "pass@8": 0.9998572295938166, + "pass@16": 0.9999945349140869, + "pass@32": 0.9999999969725001, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9978297937229678, + "pass@4": 0.9984739589769589, + "pass@8": 0.9991927685739189, + "pass@16": 0.9997893254704586, + "pass@32": 0.999989840105361, + "pass@64": 0.9999999973246758, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9973100673652695, + "pass@2": 0.9978701553585743, + "pass@4": 0.9984959510113631, + "pass@8": 0.9992628054872507, + "pass@16": 0.9998367378399854, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9770566641566265, + "pass@2": 0.9841648829570252, + "pass@4": 0.989164249235412, + "pass@8": 0.9922976666641117, + "pass@16": 0.9941890352635745, + "pass@32": 0.9952655846372421, + "pass@64": 0.9962408325530068, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9620640673981191, + "pass@2": 0.9773606620097254, + "pass@4": 0.9866140545127782, + "pass@8": 0.9920244775588233, + "pass@16": 0.9951791105256566, + "pass@32": 0.9972397387226369, + "pass@64": 0.9990357602586, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9582020308123249, + "pass@2": 0.9730195720020293, + "pass@4": 0.9829933805543213, + "pass@8": 0.9895737184859488, + "pass@16": 0.9944316023575817, + "pass@32": 0.9979028151511814, + "pass@64": 0.9996096734768417, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9545869883040936, + "pass@2": 0.9759829569001243, + "pass@4": 0.9895990122945156, + "pass@8": 0.9958761100329401, + "pass@16": 0.9982192698442767, + "pass@32": 0.9995253069447078, + "pass@64": 0.9999798903618302, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.964654356060606, + "pass@2": 0.9848753280839894, + "pass@4": 0.994307931394939, + "pass@8": 0.9984355730943542, + "pass@16": 0.9997865737709953, + "pass@32": 0.999996225078149, + "pass@64": 0.9999999999355342, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9025106837606838, + "pass@2": 0.9249840865244409, + "pass@4": 0.9394740470475377, + "pass@8": 0.951603221594124, + "pass@16": 0.9618929894371758, + "pass@32": 0.9704062836352245, + "pass@64": 0.9772483460299233, + "pass@128": 0.9829059829059829 + }, + "17": { + "pass@1": 0.8692498059006211, + "pass@2": 0.9090858377757125, + "pass@4": 0.9353135909447651, + "pass@8": 0.9535516185363221, + "pass@16": 0.965466357509199, + "pass@32": 0.9717789256494813, + "pass@64": 0.9742932379404277, + "pass@128": 0.9751552795031055 + }, + "18": { + "pass@1": 0.8068244485294118, + "pass@2": 0.8750351001621121, + "pass@4": 0.9166487397714989, + "pass@8": 0.9411302798270424, + "pass@16": 0.9559175964520414, + "pass@32": 0.9659157855028336, + "pass@64": 0.9733500817730232, + "pass@128": 0.9794117647058823 + }, + "19": { + "pass@1": 0.6194428943452381, + "pass@2": 0.7078185051087363, + "pass@4": 0.7742094414537468, + "pass@8": 0.8272903079106774, + "pass@16": 0.8697325655416732, + "pass@32": 0.9032315125581434, + "pass@64": 0.9280735994123079, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.47754491017964074, + "pass@2": 0.5815400449101793, + "pass@4": 0.6627141611041132, + "pass@8": 0.7251319869091699, + "pass@16": 0.7741932335037112, + "pass@32": 0.8160539579293532, + "pass@64": 0.8519449321090409, + "pass@128": 0.8802395209580839 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 733287, + "answer_accuracy": 0.9152907313468606, + "avg_response_len": 310.44483568661127, + "resp_tokens_sum": 248713501, + "pass_at_k": { + "pass@1": 0.9152907313468606, + "pass@2": 0.9382467396555126, + "pass@4": 0.9536234835709834, + "pass@8": 0.9645607797964179, + "pass@16": 0.9724503971693556, + "pass@32": 0.9781535877149617, + "pass@64": 0.9824174417387895, + "pass@128": 0.9854609362517974 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9575093283582089, + "pass@2": 0.9675553090844993, + "pass@4": 0.9764339019189767, + "pass@8": 0.9840698697287039, + "pass@16": 0.9892746985021801, + "pass@32": 0.9922997890645133, + "pass@64": 0.9946955387643621, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988383746355685, + "pass@2": 0.9995426671793577, + "pass@4": 0.9999317049654508, + "pass@8": 0.999998701511788, + "pass@16": 0.9999999997759129, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998629385964912, + "pass@2": 0.9999964025878345, + "pass@4": 0.9999999986295574, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906020220588235, + "pass@2": 0.9926593619731356, + "pass@4": 0.9945639799620636, + "pass@8": 0.9960446992193006, + "pass@16": 0.9968320635603195, + "pass@32": 0.997048210623365, + "pass@64": 0.9970588207360587, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9999753548895899, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9933494857594937, + "pass@2": 0.9949833518140139, + "pass@4": 0.9964600671949234, + "pass@8": 0.9976420959829561, + "pass@16": 0.9985911723443348, + "pass@32": 0.9994590693089006, + "pass@64": 0.9999562541175095, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913194444444444, + "pass@2": 0.9966260551890473, + "pass@4": 0.9991402473902474, + "pass@8": 0.9999150306942846, + "pass@16": 0.9999991362232366, + "pass@32": 0.9999999999722541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943478697749196, + "pass@2": 0.9973257462592097, + "pass@4": 0.9988976594967428, + "pass@8": 0.9997409379688414, + "pass@16": 0.9999841784174797, + "pass@32": 0.999999971428724, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9710813492063493, + "pass@2": 0.9827689507561553, + "pass@4": 0.9910190773772323, + "pass@8": 0.9964069864008952, + "pass@16": 0.9991975166401181, + "pass@32": 0.9999605809858803, + "pass@64": 0.9999999808615654, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9651662844036697, + "pass@2": 0.9830220297864142, + "pass@4": 0.9943498286911382, + "pass@8": 0.9989944241978972, + "pass@16": 0.9999475847370622, + "pass@32": 0.9999998481010195, + "pass@64": 0.9999999999999678, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9363425925925926, + "pass@2": 0.9636441929133857, + "pass@4": 0.9812329428150182, + "pass@8": 0.9909109265469578, + "pass@16": 0.995563713627039, + "pass@32": 0.997463540610219, + "pass@64": 0.9984545373712423, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9556982566765578, + "pass@2": 0.971536645131896, + "pass@4": 0.9815543184142042, + "pass@8": 0.9879400706835424, + "pass@16": 0.9919984009149101, + "pass@32": 0.9955077699507003, + "pass@64": 0.9984439818222332, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9340036651234568, + "pass@2": 0.9567897437299503, + "pass@4": 0.9704516715503154, + "pass@8": 0.9800978369588287, + "pass@16": 0.9872273677362894, + "pass@32": 0.9925171450697239, + "pass@64": 0.9958359460135142, + "pass@128": 0.9969135802469136 + }, + "16": { + "pass@1": 0.8969625737463127, + "pass@2": 0.9258290653380717, + "pass@4": 0.9432881274243372, + "pass@8": 0.9546124219600274, + "pass@16": 0.9626515201381516, + "pass@32": 0.9696621383207606, + "pass@64": 0.9756120011655725, + "pass@128": 0.9793510324483776 + }, + "17": { + "pass@1": 0.8387142319277109, + "pass@2": 0.8761247005739499, + "pass@4": 0.9008967927390097, + "pass@8": 0.9209284437120231, + "pass@16": 0.937609642297589, + "pass@32": 0.9496274880196173, + "pass@64": 0.9569150319903035, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.819304435483871, + "pass@2": 0.8832704946659896, + "pass@4": 0.9202918151360103, + "pass@8": 0.9459392546840153, + "pass@16": 0.9635935725119584, + "pass@32": 0.9729778671198072, + "pass@64": 0.9781526163741832, + "pass@128": 0.9838709677419355 + }, + "19": { + "pass@1": 0.6321988293051359, + "pass@2": 0.7248134827651832, + "pass@4": 0.7858956545650826, + "pass@8": 0.828820803601541, + "pass@16": 0.8605542666697465, + "pass@32": 0.8848513260783547, + "pass@64": 0.905053052393738, + "pass@128": 0.9214501510574018 + }, + "20": { + "pass@1": 0.5138419669669669, + "pass@2": 0.6109437439407909, + "pass@4": 0.6861744289283656, + "pass@8": 0.7468378163580444, + "pass@16": 0.7959367036137914, + "pass@32": 0.8357036484296656, + "pass@64": 0.8675307030307697, + "pass@128": 0.8888888888888888 + } + } + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl b/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1da8c41dfddfac38ac6dc532ac099f61378e263d --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb18d559562580a79812b126ac3310f0999c5cc942f0954d4c7b557696c292e3 +size 2708951048 diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d1f5922aae5b544cfc49964d8c89d3a2ffff9958 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ab4985f208a74ba69c0cafcd566c44396812b21d93da2410cc3bacad99f1ab +size 373215241 diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..304a518d8272e83f51dc8bb67fa8a0397bdf8d9d --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e8a42cbbf5bba96958a9c3cf17f5ddbfc0a59a3aa61c3da10ef4fa15cd02490 +size 423543596 diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/composition_score_report.jsonl b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3dd5a14c43d470f0b342090aec42fca2cf8afcc9 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition", "results_dir": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024156, "value_mismatch": 9626, "dependency_mismatch": 126467, "answer_mismatch": 8447, "extra_nodes": 11789, "missing_nodes": 9372, "perfect_with_extra": 832, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337828, "dependency_mismatch": 40795, "value_mismatch": 2837, "answer_mismatch": 2396, "extra_nodes": 3565, "missing_nodes": 2829, "perfect_with_extra": 194}, "teachers_in_school": {"total": 390656, "perfect": 346878, "value_mismatch": 3053, "dependency_mismatch": 43324, "answer_mismatch": 2898, "extra_nodes": 3875, "perfect_with_extra": 245, "missing_nodes": 3417}, "movie_festival_awards": {"total": 382336, "perfect": 339450, "dependency_mismatch": 42348, "missing_nodes": 3126, "extra_nodes": 4349, "perfect_with_extra": 393, "value_mismatch": 3736, "answer_mismatch": 3153}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 619058, "value_mismatch": 253560, "dependency_mismatch": 638256, "answer_mismatch": 198821, "extra_nodes": 43329, "missing_nodes": 193491, "perfect_with_extra": 424, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 206999, "extra_nodes": 16615, "perfect_with_extra": 50, "value_mismatch": 87251, "dependency_mismatch": 215764, "answer_mismatch": 69314, "missing_nodes": 70355}, "movie_festival_awards": {"total": 418816, "perfect": 204316, "value_mismatch": 82684, "dependency_mismatch": 206999, "answer_mismatch": 64712, "extra_nodes": 14071, "missing_nodes": 59687, "perfect_with_extra": 220}, "teachers_in_school": {"total": 430464, "perfect": 207743, "value_mismatch": 83625, "dependency_mismatch": 215493, "answer_mismatch": 64795, "missing_nodes": 63449, "extra_nodes": 12643, "perfect_with_extra": 154}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1643214, "value_mismatch": 263186, "dependency_mismatch": 764723, "answer_mismatch": 207268, "extra_nodes": 55118, "missing_nodes": 202863, "perfect_with_extra": 1256, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 544827, "dependency_mismatch": 256559, "value_mismatch": 90088, "answer_mismatch": 71710, "extra_nodes": 20180, "missing_nodes": 73184, "perfect_with_extra": 244}, "teachers_in_school": {"total": 821120, "perfect": 554621, "value_mismatch": 86678, "dependency_mismatch": 258817, "answer_mismatch": 67693, "extra_nodes": 16518, "perfect_with_extra": 399, "missing_nodes": 66866}, "movie_festival_awards": {"total": 801152, "perfect": 543766, "dependency_mismatch": 249347, "missing_nodes": 62813, "extra_nodes": 18420, "perfect_with_extra": 613, "value_mismatch": 86420, "answer_mismatch": 67865}}}} diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/results.json b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..0325715a4db133bb66d19f474013db05527d1a47 --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-38645_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.996875, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9619047619047619, + "pass@2": 0.9746031746031746, + "pass@4": 0.9873015873015873, + "pass@8": 0.9968253968253968, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9550898203592815, + "pass@2": 0.9700598802395209, + "pass@4": 0.9820359281437125, + "pass@8": 0.9880239520958084, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9053254437869822, + "pass@2": 0.9289940828402367, + "pass@4": 0.9615384615384616, + "pass@8": 0.9881656804733728, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8141025641025641, + "pass@2": 0.875, + "pass@4": 0.9391025641025641, + "pass@8": 0.9743589743589743, + "pass@16": 0.9935897435897436, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7577464788732394, + "pass@2": 0.8422535211267606, + "pass@4": 0.9183098591549296, + "pass@8": 0.9661971830985916, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6578171091445427, + "pass@2": 0.7109144542772862, + "pass@4": 0.8082595870206489, + "pass@8": 0.8672566371681416, + "pass@16": 0.9203539823008849, + "pass@32": 0.9469026548672567, + "pass@64": 0.9587020648967551, + "pass@128": 0.976401179941003 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9619047619047619, + "6": 0.9550898203592815, + "7": 0.9053254437869822, + "8": 0.8141025641025641, + "9": 0.7577464788732394, + "10": 0.6578171091445427 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9746031746031746, + "6": 0.9700598802395209, + "7": 0.9289940828402367, + "8": 0.875, + "9": 0.8422535211267606, + "10": 0.7109144542772862 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9820359281437125, + "7": 0.9615384615384616, + "8": 0.9391025641025641, + "9": 0.9183098591549296, + "10": 0.8082595870206489 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9880239520958084, + "7": 0.9881656804733728, + "8": 0.9743589743589743, + "9": 0.9661971830985916, + "10": 0.8672566371681416 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9887323943661972, + "10": 0.9203539823008849 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9469026548672567 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9587020648967551 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.976401179941003 + } + }, + "total_pass_at_k": { + "pass@1": 0.8912529550827423, + "pass@2": 0.9202971968929415, + "pass@4": 0.9540695710908477, + "pass@8": 0.9750084430935495, + "pass@16": 0.9885173927727119, + "pass@32": 0.9935832489023978, + "pass@64": 0.9952718676122931, + "pass@128": 0.9972982100641675 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9382352941176471, + "pass@2": 0.9647058823529412, + "pass@4": 0.9852941176470589, + "pass@8": 0.9911764705882353, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9684542586750788, + "pass@2": 0.9716088328075709, + "pass@4": 0.9842271293375394, + "pass@8": 0.9873817034700315, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.870253164556962, + "pass@2": 0.9050632911392406, + "pass@4": 0.9525316455696202, + "pass@8": 0.9810126582278481, + "pass@16": 0.9968354430379747, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7927927927927928, + "pass@2": 0.8738738738738738, + "pass@4": 0.9429429429429429, + "pass@8": 0.9819819819819819, + "pass@16": 0.993993993993994, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.77491961414791, + "pass@2": 0.864951768488746, + "pass@4": 0.9131832797427653, + "pass@8": 0.9646302250803859, + "pass@16": 0.9839228295819936, + "pass@32": 0.9935691318327974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5880597014925373, + "pass@2": 0.6686567164179105, + "pass@4": 0.7820895522388059, + "pass@8": 0.8597014925373134, + "pass@16": 0.9164179104477612, + "pass@32": 0.9492537313432836, + "pass@64": 0.9761194029850746, + "pass@128": 0.9880597014925373 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9382352941176471, + "6": 0.9684542586750788, + "7": 0.870253164556962, + "8": 0.7927927927927928, + "9": 0.77491961414791, + "10": 0.5880597014925373 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9647058823529412, + "6": 0.9716088328075709, + "7": 0.9050632911392406, + "8": 0.8738738738738738, + "9": 0.864951768488746, + "10": 0.6686567164179105 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9842271293375394, + "7": 0.9525316455696202, + "8": 0.9429429429429429, + "9": 0.9131832797427653, + "10": 0.7820895522388059 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9911764705882353, + "6": 0.9873817034700315, + "7": 0.9810126582278481, + "8": 0.9819819819819819, + "9": 0.9646302250803859, + "10": 0.8597014925373134 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.993993993993994, + "9": 0.9839228295819936, + "10": 0.9164179104477612 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9935691318327974, + "10": 0.9492537313432836 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 1.0, + "10": 0.9761194029850746 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9880597014925373 + } + }, + "total_pass_at_k": { + "pass@1": 0.8818212253096752, + "pass@2": 0.916973552058922, + "pass@4": 0.9517910947438902, + "pass@8": 0.9742216270505524, + "pass@16": 0.987612989621694, + "pass@32": 0.9929695346501507, + "pass@64": 0.9966521593572146, + "pass@128": 0.9983260796786073 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9763313609467456, + "pass@2": 0.9822485207100592, + "pass@4": 0.9940828402366864, + "pass@8": 0.9970414201183432, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9478260869565217, + "pass@2": 0.9681159420289855, + "pass@4": 0.9884057971014493, + "pass@8": 0.9942028985507246, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9799426934097422, + "pass@2": 0.9914040114613181, + "pass@4": 0.9914040114613181, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8670520231213873, + "pass@2": 0.9132947976878613, + "pass@4": 0.9450867052023122, + "pass@8": 0.9797687861271677, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8169014084507042, + "pass@2": 0.8591549295774648, + "pass@4": 0.9211267605633803, + "pass@8": 0.9887323943661972, + "pass@16": 0.9943661971830986, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7395209580838323, + "pass@2": 0.8023952095808383, + "pass@4": 0.8772455089820359, + "pass@8": 0.9491017964071856, + "pass@16": 0.9820359281437125, + "pass@32": 0.9940119760479041, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6625766871165644, + "pass@2": 0.7055214723926381, + "pass@4": 0.7975460122699386, + "pass@8": 0.8773006134969326, + "pass@16": 0.9202453987730062, + "pass@32": 0.941717791411043, + "pass@64": 0.941717791411043, + "pass@128": 0.9478527607361963 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9763313609467456, + "5": 0.9478260869565217, + "6": 0.9799426934097422, + "7": 0.8670520231213873, + "8": 0.8169014084507042, + "9": 0.7395209580838323, + "10": 0.6625766871165644 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9681159420289855, + "6": 0.9914040114613181, + "7": 0.9132947976878613, + "8": 0.8591549295774648, + "9": 0.8023952095808383, + "10": 0.7055214723926381 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9884057971014493, + "6": 0.9914040114613181, + "7": 0.9450867052023122, + "8": 0.9211267605633803, + "9": 0.8772455089820359, + "10": 0.7975460122699386 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9942028985507246, + "6": 1.0, + "7": 0.9797687861271677, + "8": 0.9887323943661972, + "9": 0.9491017964071856, + "10": 0.8773006134969326 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9820359281437125, + "10": 0.9202453987730062 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.941717791411043 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.941717791411043 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9478527607361963 + } + }, + "total_pass_at_k": { + "pass@1": 0.8882699868938401, + "pass@2": 0.9141546526867628, + "pass@4": 0.9465923984272608, + "pass@8": 0.9767365661861075, + "pass@16": 0.9882044560943644, + "pass@32": 0.9927916120576671, + "pass@64": 0.9934469200524246, + "pass@128": 0.994429882044561 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985, + "pass@2": 0.991, + "pass@4": 0.998, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.949, + "pass@2": 0.969, + "pass@4": 0.987, + "pass@8": 0.994, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.968, + "pass@2": 0.978, + "pass@4": 0.986, + "pass@8": 0.992, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.881, + "pass@2": 0.916, + "pass@4": 0.953, + "pass@8": 0.983, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.808, + "pass@2": 0.869, + "pass@4": 0.934, + "pass@8": 0.982, + "pass@16": 0.994, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.757, + "pass@2": 0.836, + "pass@4": 0.903, + "pass@8": 0.96, + "pass@16": 0.985, + "pass@32": 0.995, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.636, + "pass@2": 0.695, + "pass@4": 0.796, + "pass@8": 0.868, + "pass@16": 0.919, + "pass@32": 0.946, + "pass@64": 0.959, + "pass@128": 0.971 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985, + "5": 0.949, + "6": 0.968, + "7": 0.881, + "8": 0.808, + "9": 0.757, + "10": 0.636 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.991, + "5": 0.969, + "6": 0.978, + "7": 0.916, + "8": 0.869, + "9": 0.836, + "10": 0.695 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.987, + "6": 0.986, + "7": 0.953, + "8": 0.934, + "9": 0.903, + "10": 0.796 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.994, + "6": 0.992, + "7": 0.983, + "8": 0.982, + "9": 0.96, + "10": 0.868 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.997, + "8": 0.994, + "9": 0.985, + "10": 0.919 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 0.995, + "10": 0.946 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 1.0, + "10": 0.959 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.971 + } + }, + "total_pass_at_k": { + "pass@1": 0.8871111111111111, + "pass@2": 0.9171111111111111, + "pass@4": 0.9507777777777778, + "pass@8": 0.9753333333333334, + "pass@16": 0.9881111111111112, + "pass@32": 0.9931111111111111, + "pass@64": 0.9951111111111111, + "pass@128": 0.9966666666666667 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-38645_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6147308781869688, + "pass@2": 0.6940509915014165, + "pass@4": 0.7903682719546742, + "pass@8": 0.8753541076487252, + "pass@16": 0.9405099150141643, + "pass@32": 0.9688385269121813, + "pass@64": 0.9801699716713881, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.6610169491525424, + "pass@2": 0.7344632768361582, + "pass@4": 0.7796610169491526, + "pass@8": 0.867231638418079, + "pass@16": 0.9265536723163842, + "pass@32": 0.9519774011299436, + "pass@64": 0.9689265536723164, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.5862068965517241, + "pass@2": 0.658307210031348, + "pass@4": 0.7272727272727273, + "pass@8": 0.7931034482758621, + "pass@16": 0.8683385579937304, + "pass@32": 0.9216300940438872, + "pass@64": 0.9435736677115988, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.5638629283489096, + "pass@2": 0.5950155763239875, + "pass@4": 0.6355140186915887, + "pass@8": 0.7040498442367601, + "pass@16": 0.7757009345794392, + "pass@32": 0.8473520249221184, + "pass@64": 0.8940809968847352, + "pass@128": 0.9376947040498442 + }, + "15": { + "pass@1": 0.5028901734104047, + "pass@2": 0.5346820809248555, + "pass@4": 0.5780346820809249, + "pass@8": 0.6560693641618497, + "pass@16": 0.7225433526011561, + "pass@32": 0.815028901734104, + "pass@64": 0.8930635838150289, + "pass@128": 0.9335260115606936 + }, + "16": { + "pass@1": 0.4935483870967742, + "pass@2": 0.5193548387096775, + "pass@4": 0.5612903225806452, + "pass@8": 0.6258064516129033, + "pass@16": 0.6516129032258065, + "pass@32": 0.7129032258064516, + "pass@64": 0.7580645161290323, + "pass@128": 0.7870967741935484 + }, + "17": { + "pass@1": 0.5057803468208093, + "pass@2": 0.546242774566474, + "pass@4": 0.5982658959537572, + "pass@8": 0.6416184971098265, + "pass@16": 0.6965317919075145, + "pass@32": 0.7630057803468208, + "pass@64": 0.7832369942196532, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.40285714285714286, + "pass@2": 0.4342857142857143, + "pass@4": 0.4657142857142857, + "pass@8": 0.52, + "pass@16": 0.5714285714285714, + "pass@32": 0.6371428571428571, + "pass@64": 0.7028571428571428, + "pass@128": 0.7542857142857143 + }, + "19": { + "pass@1": 0.3213213213213213, + "pass@2": 0.37237237237237236, + "pass@4": 0.4144144144144144, + "pass@8": 0.43843843843843844, + "pass@16": 0.4744744744744745, + "pass@32": 0.5255255255255256, + "pass@64": 0.5825825825825826, + "pass@128": 0.6336336336336337 + }, + "20": { + "pass@1": 0.1981981981981982, + "pass@2": 0.25225225225225223, + "pass@4": 0.2912912912912913, + "pass@8": 0.32432432432432434, + "pass@16": 0.3663663663663664, + "pass@32": 0.4294294294294294, + "pass@64": 0.48348348348348347, + "pass@128": 0.5315315315315315 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6147308781869688, + "12": 0.6610169491525424, + "13": 0.5862068965517241, + "14": 0.5638629283489096, + "15": 0.5028901734104047, + "16": 0.4935483870967742, + "17": 0.5057803468208093, + "18": 0.40285714285714286, + "19": 0.3213213213213213, + "20": 0.1981981981981982 + }, + "pass@2": { + "11": 0.6940509915014165, + "12": 0.7344632768361582, + "13": 0.658307210031348, + "14": 0.5950155763239875, + "15": 0.5346820809248555, + "16": 0.5193548387096775, + "17": 0.546242774566474, + "18": 0.4342857142857143, + "19": 0.37237237237237236, + "20": 0.25225225225225223 + }, + "pass@4": { + "11": 0.7903682719546742, + "12": 0.7796610169491526, + "13": 0.7272727272727273, + "14": 0.6355140186915887, + "15": 0.5780346820809249, + "16": 0.5612903225806452, + "17": 0.5982658959537572, + "18": 0.4657142857142857, + "19": 0.4144144144144144, + "20": 0.2912912912912913 + }, + "pass@8": { + "11": 0.8753541076487252, + "12": 0.867231638418079, + "13": 0.7931034482758621, + "14": 0.7040498442367601, + "15": 0.6560693641618497, + "16": 0.6258064516129033, + "17": 0.6416184971098265, + "18": 0.52, + "19": 0.43843843843843844, + "20": 0.32432432432432434 + }, + "pass@16": { + "11": 0.9405099150141643, + "12": 0.9265536723163842, + "13": 0.8683385579937304, + "14": 0.7757009345794392, + "15": 0.7225433526011561, + "16": 0.6516129032258065, + "17": 0.6965317919075145, + "18": 0.5714285714285714, + "19": 0.4744744744744745, + "20": 0.3663663663663664 + }, + "pass@32": { + "11": 0.9688385269121813, + "12": 0.9519774011299436, + "13": 0.9216300940438872, + "14": 0.8473520249221184, + "15": 0.815028901734104, + "16": 0.7129032258064516, + "17": 0.7630057803468208, + "18": 0.6371428571428571, + "19": 0.5255255255255256, + "20": 0.4294294294294294 + }, + "pass@64": { + "11": 0.9801699716713881, + "12": 0.9689265536723164, + "13": 0.9435736677115988, + "14": 0.8940809968847352, + "15": 0.8930635838150289, + "16": 0.7580645161290323, + "17": 0.7832369942196532, + "18": 0.7028571428571428, + "19": 0.5825825825825826, + "20": 0.48348348348348347 + }, + "pass@128": { + "11": 0.9801699716713881, + "12": 0.9717514124293786, + "13": 0.9498432601880877, + "14": 0.9376947040498442, + "15": 0.9335260115606936, + "16": 0.7870967741935484, + "17": 0.8352601156069365, + "18": 0.7542857142857143, + "19": 0.6336336336336337, + "20": 0.5315315315315315 + } + }, + "total_pass_at_k": { + "pass@1": 0.48588410104011887, + "pass@2": 0.5352154531946508, + "pass@4": 0.5854383358098069, + "pass@8": 0.6460624071322437, + "pass@16": 0.701040118870728, + "pass@32": 0.7586924219910847, + "pass@64": 0.8002971768202081, + "pass@128": 0.8326894502228827 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5968253968253968, + "pass@2": 0.653968253968254, + "pass@4": 0.7492063492063492, + "pass@8": 0.8507936507936508, + "pass@16": 0.9079365079365079, + "pass@32": 0.9682539682539683, + "pass@64": 0.9873015873015873, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.6697247706422018, + "pass@2": 0.72782874617737, + "pass@4": 0.7706422018348624, + "pass@8": 0.8776758409785933, + "pass@16": 0.9327217125382263, + "pass@32": 0.9785932721712538, + "pass@64": 0.9877675840978594, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.6172839506172839, + "pass@2": 0.6820987654320988, + "pass@4": 0.7623456790123457, + "pass@8": 0.8518518518518519, + "pass@16": 0.9012345679012346, + "pass@32": 0.9320987654320988, + "pass@64": 0.9629629629629629, + "pass@128": 0.9722222222222222 + }, + "14": { + "pass@1": 0.5637982195845698, + "pass@2": 0.599406528189911, + "pass@4": 0.6528189910979229, + "pass@8": 0.7002967359050445, + "pass@16": 0.7566765578635015, + "pass@32": 0.8160237388724035, + "pass@64": 0.8991097922848664, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.49382716049382713, + "pass@2": 0.5277777777777778, + "pass@4": 0.5802469135802469, + "pass@8": 0.6450617283950617, + "pass@16": 0.7283950617283951, + "pass@32": 0.808641975308642, + "pass@64": 0.8950617283950617, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.528023598820059, + "pass@2": 0.551622418879056, + "pass@4": 0.6047197640117994, + "pass@8": 0.6430678466076696, + "pass@16": 0.6902654867256637, + "pass@32": 0.7492625368731564, + "pass@64": 0.8023598820058997, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.4819277108433735, + "pass@2": 0.49698795180722893, + "pass@4": 0.5271084337349398, + "pass@8": 0.572289156626506, + "pass@16": 0.6325301204819277, + "pass@32": 0.6837349397590361, + "pass@64": 0.7439759036144579, + "pass@128": 0.8072289156626506 + }, + "18": { + "pass@1": 0.41935483870967744, + "pass@2": 0.4774193548387097, + "pass@4": 0.5161290322580645, + "pass@8": 0.5516129032258065, + "pass@16": 0.6, + "pass@32": 0.6580645161290323, + "pass@64": 0.7419354838709677, + "pass@128": 0.7838709677419354 + }, + "19": { + "pass@1": 0.31722054380664655, + "pass@2": 0.3806646525679758, + "pass@4": 0.42900302114803623, + "pass@8": 0.46827794561933533, + "pass@16": 0.5045317220543807, + "pass@32": 0.5468277945619335, + "pass@64": 0.5921450151057401, + "pass@128": 0.6283987915407855 + }, + "20": { + "pass@1": 0.2222222222222222, + "pass@2": 0.2672672672672673, + "pass@4": 0.29429429429429427, + "pass@8": 0.34234234234234234, + "pass@16": 0.4024024024024024, + "pass@32": 0.44744744744744747, + "pass@64": 0.5015015015015015, + "pass@128": 0.5735735735735735 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5968253968253968, + "12": 0.6697247706422018, + "13": 0.6172839506172839, + "14": 0.5637982195845698, + "15": 0.49382716049382713, + "16": 0.528023598820059, + "17": 0.4819277108433735, + "18": 0.41935483870967744, + "19": 0.31722054380664655, + "20": 0.2222222222222222 + }, + "pass@2": { + "11": 0.653968253968254, + "12": 0.72782874617737, + "13": 0.6820987654320988, + "14": 0.599406528189911, + "15": 0.5277777777777778, + "16": 0.551622418879056, + "17": 0.49698795180722893, + "18": 0.4774193548387097, + "19": 0.3806646525679758, + "20": 0.2672672672672673 + }, + "pass@4": { + "11": 0.7492063492063492, + "12": 0.7706422018348624, + "13": 0.7623456790123457, + "14": 0.6528189910979229, + "15": 0.5802469135802469, + "16": 0.6047197640117994, + "17": 0.5271084337349398, + "18": 0.5161290322580645, + "19": 0.42900302114803623, + "20": 0.29429429429429427 + }, + "pass@8": { + "11": 0.8507936507936508, + "12": 0.8776758409785933, + "13": 0.8518518518518519, + "14": 0.7002967359050445, + "15": 0.6450617283950617, + "16": 0.6430678466076696, + "17": 0.572289156626506, + "18": 0.5516129032258065, + "19": 0.46827794561933533, + "20": 0.34234234234234234 + }, + "pass@16": { + "11": 0.9079365079365079, + "12": 0.9327217125382263, + "13": 0.9012345679012346, + "14": 0.7566765578635015, + "15": 0.7283950617283951, + "16": 0.6902654867256637, + "17": 0.6325301204819277, + "18": 0.6, + "19": 0.5045317220543807, + "20": 0.4024024024024024 + }, + "pass@32": { + "11": 0.9682539682539683, + "12": 0.9785932721712538, + "13": 0.9320987654320988, + "14": 0.8160237388724035, + "15": 0.808641975308642, + "16": 0.7492625368731564, + "17": 0.6837349397590361, + "18": 0.6580645161290323, + "19": 0.5468277945619335, + "20": 0.44744744744744747 + }, + "pass@64": { + "11": 0.9873015873015873, + "12": 0.9877675840978594, + "13": 0.9629629629629629, + "14": 0.8991097922848664, + "15": 0.8950617283950617, + "16": 0.8023598820058997, + "17": 0.7439759036144579, + "18": 0.7419354838709677, + "19": 0.5921450151057401, + "20": 0.5015015015015015 + }, + "pass@128": { + "11": 0.9968253968253968, + "12": 0.9908256880733946, + "13": 0.9722222222222222, + "14": 0.9228486646884273, + "15": 0.9290123456790124, + "16": 0.8259587020648967, + "17": 0.8072289156626506, + "18": 0.7838709677419354, + "19": 0.6283987915407855, + "20": 0.5735735735735735 + } + }, + "total_pass_at_k": { + "pass@1": 0.49052567237163813, + "pass@2": 0.535757946210269, + "pass@4": 0.5877139364303179, + "pass@8": 0.6491442542787286, + "pass@16": 0.7044621026894865, + "pass@32": 0.757640586797066, + "pass@64": 0.8102078239608802, + "pass@128": 0.8419926650366748 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6566265060240963, + "pass@2": 0.6957831325301205, + "pass@4": 0.7620481927710844, + "pass@8": 0.8524096385542169, + "pass@16": 0.9096385542168675, + "pass@32": 0.9457831325301205, + "pass@64": 0.9668674698795181, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.5987460815047022, + "pass@2": 0.6551724137931034, + "pass@4": 0.7210031347962382, + "pass@8": 0.8181818181818182, + "pass@16": 0.9090909090909091, + "pass@32": 0.9498432601880877, + "pass@64": 0.9686520376175548, + "pass@128": 0.9843260188087775 + }, + "13": { + "pass@1": 0.6246498599439776, + "pass@2": 0.6890756302521008, + "pass@4": 0.7759103641456583, + "pass@8": 0.8487394957983193, + "pass@16": 0.9019607843137255, + "pass@32": 0.938375350140056, + "pass@64": 0.9607843137254902, + "pass@128": 0.9747899159663865 + }, + "14": { + "pass@1": 0.5818713450292398, + "pass@2": 0.6198830409356725, + "pass@4": 0.6666666666666666, + "pass@8": 0.7309941520467836, + "pass@16": 0.7953216374269005, + "pass@32": 0.8567251461988304, + "pass@64": 0.9152046783625731, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.5787878787878787, + "pass@2": 0.6060606060606061, + "pass@4": 0.6393939393939394, + "pass@8": 0.693939393939394, + "pass@16": 0.7484848484848485, + "pass@32": 0.8181818181818182, + "pass@64": 0.8848484848484849, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5071225071225072, + "pass@2": 0.5584045584045584, + "pass@4": 0.5925925925925926, + "pass@8": 0.6438746438746439, + "pass@16": 0.7008547008547008, + "pass@32": 0.7720797720797721, + "pass@64": 0.8176638176638177, + "pass@128": 0.8347578347578347 + }, + "17": { + "pass@1": 0.40993788819875776, + "pass@2": 0.45652173913043476, + "pass@4": 0.4937888198757764, + "pass@8": 0.5434782608695652, + "pass@16": 0.6242236024844721, + "pass@32": 0.7018633540372671, + "pass@64": 0.7391304347826086, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.3911764705882353, + "pass@2": 0.4294117647058823, + "pass@4": 0.4647058823529412, + "pass@8": 0.49411764705882355, + "pass@16": 0.5294117647058824, + "pass@32": 0.5941176470588235, + "pass@64": 0.6705882352941176, + "pass@128": 0.7382352941176471 + }, + "19": { + "pass@1": 0.31845238095238093, + "pass@2": 0.38392857142857145, + "pass@4": 0.43452380952380953, + "pass@8": 0.46130952380952384, + "pass@16": 0.5148809523809523, + "pass@32": 0.5505952380952381, + "pass@64": 0.6130952380952381, + "pass@128": 0.6577380952380952 + }, + "20": { + "pass@1": 0.19461077844311378, + "pass@2": 0.24550898203592814, + "pass@4": 0.2844311377245509, + "pass@8": 0.31736526946107785, + "pass@16": 0.3562874251497006, + "pass@32": 0.40119760479041916, + "pass@64": 0.45808383233532934, + "pass@128": 0.5239520958083832 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6566265060240963, + "12": 0.5987460815047022, + "13": 0.6246498599439776, + "14": 0.5818713450292398, + "15": 0.5787878787878787, + "16": 0.5071225071225072, + "17": 0.40993788819875776, + "18": 0.3911764705882353, + "19": 0.31845238095238093, + "20": 0.19461077844311378 + }, + "pass@2": { + "11": 0.6957831325301205, + "12": 0.6551724137931034, + "13": 0.6890756302521008, + "14": 0.6198830409356725, + "15": 0.6060606060606061, + "16": 0.5584045584045584, + "17": 0.45652173913043476, + "18": 0.4294117647058823, + "19": 0.38392857142857145, + "20": 0.24550898203592814 + }, + "pass@4": { + "11": 0.7620481927710844, + "12": 0.7210031347962382, + "13": 0.7759103641456583, + "14": 0.6666666666666666, + "15": 0.6393939393939394, + "16": 0.5925925925925926, + "17": 0.4937888198757764, + "18": 0.4647058823529412, + "19": 0.43452380952380953, + "20": 0.2844311377245509 + }, + "pass@8": { + "11": 0.8524096385542169, + "12": 0.8181818181818182, + "13": 0.8487394957983193, + "14": 0.7309941520467836, + "15": 0.693939393939394, + "16": 0.6438746438746439, + "17": 0.5434782608695652, + "18": 0.49411764705882355, + "19": 0.46130952380952384, + "20": 0.31736526946107785 + }, + "pass@16": { + "11": 0.9096385542168675, + "12": 0.9090909090909091, + "13": 0.9019607843137255, + "14": 0.7953216374269005, + "15": 0.7484848484848485, + "16": 0.7008547008547008, + "17": 0.6242236024844721, + "18": 0.5294117647058824, + "19": 0.5148809523809523, + "20": 0.3562874251497006 + }, + "pass@32": { + "11": 0.9457831325301205, + "12": 0.9498432601880877, + "13": 0.938375350140056, + "14": 0.8567251461988304, + "15": 0.8181818181818182, + "16": 0.7720797720797721, + "17": 0.7018633540372671, + "18": 0.5941176470588235, + "19": 0.5505952380952381, + "20": 0.40119760479041916 + }, + "pass@64": { + "11": 0.9668674698795181, + "12": 0.9686520376175548, + "13": 0.9607843137254902, + "14": 0.9152046783625731, + "15": 0.8848484848484849, + "16": 0.8176638176638177, + "17": 0.7391304347826086, + "18": 0.6705882352941176, + "19": 0.6130952380952381, + "20": 0.45808383233532934 + }, + "pass@128": { + "11": 0.9789156626506024, + "12": 0.9843260188087775, + "13": 0.9747899159663865, + "14": 0.9678362573099415, + "15": 0.9515151515151515, + "16": 0.8347578347578347, + "17": 0.8043478260869565, + "18": 0.7382352941176471, + "19": 0.6577380952380952, + "20": 0.5239520958083832 + } + }, + "total_pass_at_k": { + "pass@1": 0.48676776687481416, + "pass@2": 0.5346416889681832, + "pass@4": 0.584299732381802, + "pass@8": 0.6410942610764199, + "pass@16": 0.6993755575379126, + "pass@32": 0.7531965506987809, + "pass@64": 0.79988105857865, + "pass@128": 0.8418079096045198 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.623, + "pass@2": 0.682, + "pass@4": 0.768, + "pass@8": 0.86, + "pass@16": 0.92, + "pass@32": 0.961, + "pass@64": 0.978, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.644, + "pass@2": 0.707, + "pass@4": 0.758, + "pass@8": 0.855, + "pass@16": 0.923, + "pass@32": 0.96, + "pass@64": 0.975, + "pass@128": 0.982 + }, + "13": { + "pass@1": 0.61, + "pass@2": 0.677, + "pass@4": 0.756, + "pass@8": 0.832, + "pass@16": 0.891, + "pass@32": 0.931, + "pass@64": 0.956, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.57, + "pass@2": 0.605, + "pass@4": 0.652, + "pass@8": 0.712, + "pass@16": 0.776, + "pass@32": 0.84, + "pass@64": 0.903, + "pass@128": 0.943 + }, + "15": { + "pass@1": 0.525, + "pass@2": 0.556, + "pass@4": 0.599, + "pass@8": 0.665, + "pass@16": 0.733, + "pass@32": 0.814, + "pass@64": 0.891, + "pass@128": 0.938 + }, + "16": { + "pass@1": 0.51, + "pass@2": 0.544, + "pass@4": 0.587, + "pass@8": 0.638, + "pass@16": 0.682, + "pass@32": 0.746, + "pass@64": 0.794, + "pass@128": 0.817 + }, + "17": { + "pass@1": 0.467, + "pass@2": 0.501, + "pass@4": 0.541, + "pass@8": 0.587, + "pass@16": 0.652, + "pass@32": 0.717, + "pass@64": 0.756, + "pass@128": 0.816 + }, + "18": { + "pass@1": 0.404, + "pass@2": 0.446, + "pass@4": 0.481, + "pass@8": 0.521, + "pass@16": 0.566, + "pass@32": 0.629, + "pass@64": 0.704, + "pass@128": 0.758 + }, + "19": { + "pass@1": 0.319, + "pass@2": 0.379, + "pass@4": 0.426, + "pass@8": 0.456, + "pass@16": 0.498, + "pass@32": 0.541, + "pass@64": 0.596, + "pass@128": 0.64 + }, + "20": { + "pass@1": 0.205, + "pass@2": 0.255, + "pass@4": 0.29, + "pass@8": 0.328, + "pass@16": 0.375, + "pass@32": 0.426, + "pass@64": 0.481, + "pass@128": 0.543 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.623, + "12": 0.644, + "13": 0.61, + "14": 0.57, + "15": 0.525, + "16": 0.51, + "17": 0.467, + "18": 0.404, + "19": 0.319, + "20": 0.205 + }, + "pass@2": { + "11": 0.682, + "12": 0.707, + "13": 0.677, + "14": 0.605, + "15": 0.556, + "16": 0.544, + "17": 0.501, + "18": 0.446, + "19": 0.379, + "20": 0.255 + }, + "pass@4": { + "11": 0.768, + "12": 0.758, + "13": 0.756, + "14": 0.652, + "15": 0.599, + "16": 0.587, + "17": 0.541, + "18": 0.481, + "19": 0.426, + "20": 0.29 + }, + "pass@8": { + "11": 0.86, + "12": 0.855, + "13": 0.832, + "14": 0.712, + "15": 0.665, + "16": 0.638, + "17": 0.587, + "18": 0.521, + "19": 0.456, + "20": 0.328 + }, + "pass@16": { + "11": 0.92, + "12": 0.923, + "13": 0.891, + "14": 0.776, + "15": 0.733, + "16": 0.682, + "17": 0.652, + "18": 0.566, + "19": 0.498, + "20": 0.375 + }, + "pass@32": { + "11": 0.961, + "12": 0.96, + "13": 0.931, + "14": 0.84, + "15": 0.814, + "16": 0.746, + "17": 0.717, + "18": 0.629, + "19": 0.541, + "20": 0.426 + }, + "pass@64": { + "11": 0.978, + "12": 0.975, + "13": 0.956, + "14": 0.903, + "15": 0.891, + "16": 0.794, + "17": 0.756, + "18": 0.704, + "19": 0.596, + "20": 0.481 + }, + "pass@128": { + "11": 0.985, + "12": 0.982, + "13": 0.966, + "14": 0.943, + "15": 0.938, + "16": 0.817, + "17": 0.816, + "18": 0.758, + "19": 0.64, + "20": 0.543 + } + }, + "total_pass_at_k": { + "pass@1": 0.4877, + "pass@2": 0.5352, + "pass@4": 0.5858, + "pass@8": 0.6454, + "pass@16": 0.7016, + "pass@32": 0.7565, + "pass@64": 0.8034, + "pass@128": 0.8388 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/summary.csv b/Qwen3-100M-midtrain-diff_ar-block8/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a50aedb85e6b8ef14890c6b8fe7102f422aa692e --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-38645,0.9926675347222222,0.844671875,0.9147750822368421,0.0,0.0,0.0,200.20069097222222,377.0967484375,1152000,1280000,2432000 diff --git a/Qwen3-100M-midtrain-diff_ar-block8/composition/summary.json b/Qwen3-100M-midtrain-diff_ar-block8/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b21d1ff472b1434c03ccc9660cab7d769a5b999a --- /dev/null +++ b/Qwen3-100M-midtrain-diff_ar-block8/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-38645", + "id_acc": 0.9926675347222222, + "ood_acc": 0.844671875, + "total_acc": 0.9147750822368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.20069097222222, + "ood_avg_resp_len": 377.0967484375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 738018, + "answer_accuracy": 0.9114393969332911, + "avg_response_len": 280.1749056473285, + "resp_tokens_sum": 226865466, + "pass_at_k": { + "pass@1": 0.9114393969332911, + "pass@2": 0.9346978341788974, + "pass@4": 0.9510085321100552, + "pass@8": 0.9624914679892927, + "pass@16": 0.9708110810206952, + "pass@32": 0.9770193310184669, + "pass@64": 0.981604927339997, + "pass@128": 0.9851406892190958 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9615366887905604, + "pass@2": 0.970145808654449, + "pass@4": 0.9771259859110534, + "pass@8": 0.9837142589656627, + "pass@16": 0.9897806295670789, + "pass@32": 0.9934434017257641, + "pass@64": 0.9955847044260153, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997610856269113, + "pass@2": 0.9999864552481399, + "pass@4": 0.9999999638806617, + "pass@8": 0.9999999999999809, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99931640625, + "pass@2": 0.9999350239911416, + "pass@4": 0.9999994973284589, + "pass@8": 0.999999999990623, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9972468285214349, + "pass@4": 0.9985486585010207, + "pass@8": 0.9994671341886722, + "pass@16": 0.9999236250321564, + "pass@32": 0.9999990493973, + "pass@64": 0.9999999999949286, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995555763473054, + "pass@2": 0.9999499033429206, + "pass@4": 0.9999993320445721, + "pass@8": 0.9999999999490911, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964404585798816, + "pass@2": 0.9978735207100592, + "pass@4": 0.9986706386183974, + "pass@8": 0.9994230191252826, + "pass@16": 0.9999007629980622, + "pass@32": 0.9999981593967747, + "pass@64": 0.99999999996853, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.997245592948718, + "pass@2": 0.9984009848828992, + "pass@4": 0.9992322474113812, + "pass@8": 0.9998262245762686, + "pass@16": 0.9999926289511722, + "pass@32": 0.9999999947902106, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950704225352113, + "pass@2": 0.9983818759010756, + "pass@4": 0.999779442534472, + "pass@8": 0.9999935759719568, + "pass@16": 0.9999999938621692, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9698123229461756, + "pass@2": 0.981584311079387, + "pass@4": 0.9891040887948495, + "pass@8": 0.9937797636155625, + "pass@16": 0.9970789483598115, + "pass@32": 0.999223562425918, + "pass@64": 0.9999570534856826, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9659030720338984, + "pass@2": 0.9809874632990792, + "pass@4": 0.9906215364498931, + "pass@8": 0.9956137072056146, + "pass@16": 0.9978803736651605, + "pass@32": 0.9993246177441057, + "pass@64": 0.9999568688255516, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9498922413793104, + "pass@2": 0.9708758608347934, + "pass@4": 0.9840897281060871, + "pass@8": 0.9909108589255634, + "pass@16": 0.9943999521503089, + "pass@32": 0.9963005415376021, + "pass@64": 0.9980499904733999, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9233839563862928, + "pass@2": 0.944520460789364, + "pass@4": 0.9611923819102052, + "pass@8": 0.9751079976868544, + "pass@16": 0.9861293435926446, + "pass@32": 0.9935272719026856, + "pass@64": 0.9975030863628863, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9595375722543352, + "pass@2": 0.9784015719584908, + "pass@4": 0.987961396353635, + "pass@8": 0.9923778119086278, + "pass@16": 0.9938310119401556, + "pass@32": 0.9941908530213401, + "pass@64": 0.9942196182961773, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8793850806451613, + "pass@2": 0.9133687611125221, + "pass@4": 0.9350571322737883, + "pass@8": 0.9489252481008612, + "pass@16": 0.9594660322806725, + "pass@32": 0.9677192403291304, + "pass@64": 0.9722136540329196, + "pass@128": 0.9741935483870968 + }, + "17": { + "pass@1": 0.8896992413294798, + "pass@2": 0.9255197191752769, + "pass@4": 0.9464512051600483, + "pass@8": 0.9595738090472379, + "pass@16": 0.9673121876199731, + "pass@32": 0.9725731938539892, + "pass@64": 0.9768482847119901, + "pass@128": 0.9797687861271677 + }, + "18": { + "pass@1": 0.7444419642857143, + "pass@2": 0.8136916479190104, + "pass@4": 0.8625618206652741, + "pass@8": 0.8958082898451949, + "pass@16": 0.9174575405950798, + "pass@32": 0.9337781303810576, + "pass@64": 0.947397000627292, + "pass@128": 0.9571428571428572 + }, + "19": { + "pass@1": 0.6253753753753754, + "pass@2": 0.7184758577475104, + "pass@4": 0.7833039738275956, + "pass@8": 0.8296772377534075, + "pass@16": 0.8660549147719705, + "pass@32": 0.8940814351693145, + "pass@64": 0.9133406551952218, + "pass@128": 0.9279279279279279 + }, + "20": { + "pass@1": 0.4760932807807808, + "pass@2": 0.5768932071835613, + "pass@4": 0.6601997517765237, + "pass@8": 0.7265525943478015, + "pass@16": 0.7789762742387952, + "pass@32": 0.8215439955324653, + "pass@64": 0.8572865791166149, + "pass@128": 0.8888888888888888 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 753428, + "answer_accuracy": 0.9175613795791114, + "avg_response_len": 289.52658198558066, + "resp_tokens_sum": 237736067, + "pass_at_k": { + "pass@1": 0.9175613795791114, + "pass@2": 0.9393609304901669, + "pass@4": 0.9544253548538294, + "pass@8": 0.9651838128249977, + "pass@16": 0.9731036723828017, + "pass@32": 0.9792285383144879, + "pass@64": 0.9839183408625037, + "pass@128": 0.987373343725643 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9446654524539877, + "pass@2": 0.9550259498333415, + "pass@4": 0.9634138444542589, + "pass@8": 0.970866877957459, + "pass@16": 0.9782378206275941, + "pass@32": 0.9857900637806933, + "pass@64": 0.9923585282128631, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988873106060606, + "pass@2": 0.9997856269386781, + "pass@4": 0.9999895822681255, + "pass@8": 0.9999999777033803, + "pass@16": 0.9999999999999803, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9965579710144927, + "pass@2": 0.9972633515919205, + "pass@4": 0.9975327839454851, + "pass@8": 0.9979120550709301, + "pass@16": 0.9985302315822023, + "pass@32": 0.9993303337082865, + "pass@64": 0.9999164669633687, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999915393813595, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9975162572254336, + "pass@2": 0.9985822931364069, + "pass@4": 0.9993384239181086, + "pass@8": 0.9998572295938166, + "pass@16": 0.9999945349140869, + "pass@32": 0.9999999969725001, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9978297937229678, + "pass@4": 0.9984739589769589, + "pass@8": 0.9991927685739189, + "pass@16": 0.9997893254704586, + "pass@32": 0.999989840105361, + "pass@64": 0.9999999973246758, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9973100673652695, + "pass@2": 0.9978701553585743, + "pass@4": 0.9984959510113631, + "pass@8": 0.9992628054872507, + "pass@16": 0.9998367378399854, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9770566641566265, + "pass@2": 0.9841648829570252, + "pass@4": 0.989164249235412, + "pass@8": 0.9922976666641117, + "pass@16": 0.9941890352635745, + "pass@32": 0.9952655846372421, + "pass@64": 0.9962408325530068, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9620640673981191, + "pass@2": 0.9773606620097254, + "pass@4": 0.9866140545127782, + "pass@8": 0.9920244775588233, + "pass@16": 0.9951791105256566, + "pass@32": 0.9972397387226369, + "pass@64": 0.9990357602586, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9582020308123249, + "pass@2": 0.9730195720020293, + "pass@4": 0.9829933805543213, + "pass@8": 0.9895737184859488, + "pass@16": 0.9944316023575817, + "pass@32": 0.9979028151511814, + "pass@64": 0.9996096734768417, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9545869883040936, + "pass@2": 0.9759829569001243, + "pass@4": 0.9895990122945156, + "pass@8": 0.9958761100329401, + "pass@16": 0.9982192698442767, + "pass@32": 0.9995253069447078, + "pass@64": 0.9999798903618302, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.964654356060606, + "pass@2": 0.9848753280839894, + "pass@4": 0.994307931394939, + "pass@8": 0.9984355730943542, + "pass@16": 0.9997865737709953, + "pass@32": 0.999996225078149, + "pass@64": 0.9999999999355342, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9025106837606838, + "pass@2": 0.9249840865244409, + "pass@4": 0.9394740470475377, + "pass@8": 0.951603221594124, + "pass@16": 0.9618929894371758, + "pass@32": 0.9704062836352245, + "pass@64": 0.9772483460299233, + "pass@128": 0.9829059829059829 + }, + "17": { + "pass@1": 0.8692498059006211, + "pass@2": 0.9090858377757125, + "pass@4": 0.9353135909447651, + "pass@8": 0.9535516185363221, + "pass@16": 0.965466357509199, + "pass@32": 0.9717789256494813, + "pass@64": 0.9742932379404277, + "pass@128": 0.9751552795031055 + }, + "18": { + "pass@1": 0.8068244485294118, + "pass@2": 0.8750351001621121, + "pass@4": 0.9166487397714989, + "pass@8": 0.9411302798270424, + "pass@16": 0.9559175964520414, + "pass@32": 0.9659157855028336, + "pass@64": 0.9733500817730232, + "pass@128": 0.9794117647058823 + }, + "19": { + "pass@1": 0.6194428943452381, + "pass@2": 0.7078185051087363, + "pass@4": 0.7742094414537468, + "pass@8": 0.8272903079106774, + "pass@16": 0.8697325655416732, + "pass@32": 0.9032315125581434, + "pass@64": 0.9280735994123079, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.47754491017964074, + "pass@2": 0.5815400449101793, + "pass@4": 0.6627141611041132, + "pass@8": 0.7251319869091699, + "pass@16": 0.7741932335037112, + "pass@32": 0.8160539579293532, + "pass@64": 0.8519449321090409, + "pass@128": 0.8802395209580839 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 733287, + "answer_accuracy": 0.9152907313468606, + "avg_response_len": 310.44483568661127, + "resp_tokens_sum": 248713501, + "pass_at_k": { + "pass@1": 0.9152907313468606, + "pass@2": 0.9382467396555126, + "pass@4": 0.9536234835709834, + "pass@8": 0.9645607797964179, + "pass@16": 0.9724503971693556, + "pass@32": 0.9781535877149617, + "pass@64": 0.9824174417387895, + "pass@128": 0.9854609362517974 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9575093283582089, + "pass@2": 0.9675553090844993, + "pass@4": 0.9764339019189767, + "pass@8": 0.9840698697287039, + "pass@16": 0.9892746985021801, + "pass@32": 0.9922997890645133, + "pass@64": 0.9946955387643621, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988383746355685, + "pass@2": 0.9995426671793577, + "pass@4": 0.9999317049654508, + "pass@8": 0.999998701511788, + "pass@16": 0.9999999997759129, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998629385964912, + "pass@2": 0.9999964025878345, + "pass@4": 0.9999999986295574, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906020220588235, + "pass@2": 0.9926593619731356, + "pass@4": 0.9945639799620636, + "pass@8": 0.9960446992193006, + "pass@16": 0.9968320635603195, + "pass@32": 0.997048210623365, + "pass@64": 0.9970588207360587, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9999753548895899, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9933494857594937, + "pass@2": 0.9949833518140139, + "pass@4": 0.9964600671949234, + "pass@8": 0.9976420959829561, + "pass@16": 0.9985911723443348, + "pass@32": 0.9994590693089006, + "pass@64": 0.9999562541175095, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913194444444444, + "pass@2": 0.9966260551890473, + "pass@4": 0.9991402473902474, + "pass@8": 0.9999150306942846, + "pass@16": 0.9999991362232366, + "pass@32": 0.9999999999722541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943478697749196, + "pass@2": 0.9973257462592097, + "pass@4": 0.9988976594967428, + "pass@8": 0.9997409379688414, + "pass@16": 0.9999841784174797, + "pass@32": 0.999999971428724, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9710813492063493, + "pass@2": 0.9827689507561553, + "pass@4": 0.9910190773772323, + "pass@8": 0.9964069864008952, + "pass@16": 0.9991975166401181, + "pass@32": 0.9999605809858803, + "pass@64": 0.9999999808615654, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9651662844036697, + "pass@2": 0.9830220297864142, + "pass@4": 0.9943498286911382, + "pass@8": 0.9989944241978972, + "pass@16": 0.9999475847370622, + "pass@32": 0.9999998481010195, + "pass@64": 0.9999999999999678, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9363425925925926, + "pass@2": 0.9636441929133857, + "pass@4": 0.9812329428150182, + "pass@8": 0.9909109265469578, + "pass@16": 0.995563713627039, + "pass@32": 0.997463540610219, + "pass@64": 0.9984545373712423, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9556982566765578, + "pass@2": 0.971536645131896, + "pass@4": 0.9815543184142042, + "pass@8": 0.9879400706835424, + "pass@16": 0.9919984009149101, + "pass@32": 0.9955077699507003, + "pass@64": 0.9984439818222332, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9340036651234568, + "pass@2": 0.9567897437299503, + "pass@4": 0.9704516715503154, + "pass@8": 0.9800978369588287, + "pass@16": 0.9872273677362894, + "pass@32": 0.9925171450697239, + "pass@64": 0.9958359460135142, + "pass@128": 0.9969135802469136 + }, + "16": { + "pass@1": 0.8969625737463127, + "pass@2": 0.9258290653380717, + "pass@4": 0.9432881274243372, + "pass@8": 0.9546124219600274, + "pass@16": 0.9626515201381516, + "pass@32": 0.9696621383207606, + "pass@64": 0.9756120011655725, + "pass@128": 0.9793510324483776 + }, + "17": { + "pass@1": 0.8387142319277109, + "pass@2": 0.8761247005739499, + "pass@4": 0.9008967927390097, + "pass@8": 0.9209284437120231, + "pass@16": 0.937609642297589, + "pass@32": 0.9496274880196173, + "pass@64": 0.9569150319903035, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.819304435483871, + "pass@2": 0.8832704946659896, + "pass@4": 0.9202918151360103, + "pass@8": 0.9459392546840153, + "pass@16": 0.9635935725119584, + "pass@32": 0.9729778671198072, + "pass@64": 0.9781526163741832, + "pass@128": 0.9838709677419355 + }, + "19": { + "pass@1": 0.6321988293051359, + "pass@2": 0.7248134827651832, + "pass@4": 0.7858956545650826, + "pass@8": 0.828820803601541, + "pass@16": 0.8605542666697465, + "pass@32": 0.8848513260783547, + "pass@64": 0.905053052393738, + "pass@128": 0.9214501510574018 + }, + "20": { + "pass@1": 0.5138419669669669, + "pass@2": 0.6109437439407909, + "pass@4": 0.6861744289283656, + "pass@8": 0.7468378163580444, + "pass@16": 0.7959367036137914, + "pass@32": 0.8357036484296656, + "pass@64": 0.8675307030307697, + "pass@128": 0.8888888888888888 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block4/composition/autoregressive/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl b/SDAR-100M-ep1-block4/composition/autoregressive/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..90a8cee0a6a8ec8746aa529b2e7344d06635b1f0 --- /dev/null +++ b/SDAR-100M-ep1-block4/composition/autoregressive/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000eb1dcde4d138b79c0e3f64f4a31d9789aaabbe80d63901e5a452af837e1d3 +size 373677699 diff --git a/SDAR-100M-ep1-block4/composition/low_confidence_static/block4/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl b/SDAR-100M-ep1-block4/composition/low_confidence_static/block4/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7c685dcefe389de04e9f0c5607de907829d3a136 --- /dev/null +++ b/SDAR-100M-ep1-block4/composition/low_confidence_static/block4/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3252be8e68f80fe90a9e6938192a01a7de7fbcf80f1c1681e81ed9433a4c3c27 +size 373567154 diff --git a/SDAR-100M-ep1-block4/composition/low_confidence_static/block8/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl b/SDAR-100M-ep1-block4/composition/low_confidence_static/block8/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e9a4084d96442e0689ed7311a6285458f4ac9d7c --- /dev/null +++ b/SDAR-100M-ep1-block4/composition/low_confidence_static/block8/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91fc0584b21a85e91e93187343d151368b56d7b5381b0d67c3c92d28e92564b1 +size 387814950 diff --git a/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_ood_generations.jsonl b/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..26801da1fa8377af6e144045397b202f57ee3c73 --- /dev/null +++ b/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87fa99752c26a67c5e5dd73a9d822a54a319185696fc14b70ca6edaf9011f0b0 +size 2595827299 diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_metrics.json b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..38251fca677a093cf30a6cdaf220e5d142bafc99 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-39470", + "id": { + "answer_accuracy": 0.9875668402777777, + "avg_response_len": 200.47587413194444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9875668402777777, + "pass@2": 0.9928982666229137, + "pass@4": 0.9949400601174857, + "pass@8": 0.9961314883779663, + "pass@16": 0.9970118321846139, + "pass@32": 0.9977556669577146, + "pass@64": 0.9983711607952549, + "pass@128": 0.9987777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.934125, + "pass@2": 0.9531967273622055, + "pass@4": 0.9647547664041997, + "pass@8": 0.9726677273810179, + "pass@16": 0.9786270231393196, + "pass@32": 0.9837944277629593, + "pass@64": 0.988220290730658, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.99978125, + "pass@2": 0.999998031496063, + "pass@4": 0.9999999999062618, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982890625, + "pass@2": 0.999478592519685, + "pass@4": 0.9998916596362954, + "pass@8": 0.9999946416322282, + "pass@16": 0.9999999899791028, + "pass@32": 0.9999999999999952, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968125, + "pass@2": 0.9989414370078745, + "pass@4": 0.9996005144356952, + "pass@8": 0.9998957304072887, + "pass@16": 0.9999914984966795, + "pass@32": 0.9999999700068067, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9944921875, + "pass@2": 0.9977639025590551, + "pass@4": 0.9990230458380202, + "pass@8": 0.9995578080577183, + "pass@16": 0.9998392560072565, + "pass@32": 0.9999809346029047, + "pass@64": 0.9999999379486068, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9979921875, + "pass@2": 0.9998559301181105, + "pass@4": 0.9999961975065618, + "pass@8": 0.9999999962220367, + "pass@16": 0.9999999999999993, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.994859375, + "pass@2": 0.9982384350393703, + "pass@4": 0.9988248165541807, + "pass@8": 0.9989864571754808, + "pass@16": 0.9989999401575637, + "pass@32": 0.9989999999997898, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9863828125, + "pass@2": 0.9941416092519691, + "pass@4": 0.996104038901387, + "pass@8": 0.9972324929602487, + "pass@16": 0.9983763792634263, + "pass@32": 0.9993417616389402, + "pass@64": 0.9998682522170986, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9853671875, + "pass@2": 0.994469734251969, + "pass@4": 0.9962655018747654, + "pass@8": 0.9968485415656794, + "pass@16": 0.9972724026181798, + "pass@32": 0.9976839086080339, + "pass@64": 0.9982519662609338, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.934125, + "2": 0.99978125, + "3": 0.9982890625, + "4": 0.9968125, + "5": 0.9944921875, + "6": 0.9979921875, + "7": 0.994859375, + "8": 0.9863828125, + "9": 0.9853671875 + }, + "per_op_avg_response_len": { + "10": 281.61803125, + "2": 115.681953125, + "3": 145.0216015625, + "4": 160.1766953125, + "5": 173.8655703125, + "6": 199.1715, + "7": 224.4758125, + "8": 244.33884375, + "9": 259.932859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.61803125, + "2": 115.681953125, + "3": 145.0216015625, + "4": 160.1766953125, + "5": 173.8655703125, + "6": 199.1715, + "7": 224.4758125, + "8": 244.33884375, + "9": 259.932859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373929, + "answer_accuracy": 0.9865992274569402, + "avg_response_len": 190.89281228892267, + "resp_tokens_sum": 72349903, + "pass_at_k": { + "pass@1": 0.9865992274569402, + "pass@2": 0.9928458493885082, + "pass@4": 0.9949435897197256, + "pass@8": 0.9960763945899138, + "pass@16": 0.9969657892415204, + "pass@32": 0.9978278175668144, + "pass@64": 0.9986841405350807, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9300562315634219, + "pass@2": 0.9487169157782271, + "pass@4": 0.9599396154020569, + "pass@8": 0.9683942194420995, + "pass@16": 0.9757538297615362, + "pass@32": 0.9826819806419275, + "pass@64": 0.9892382643670711, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997079439252337, + "pass@2": 0.9999984668972454, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9987098623853211, + "pass@2": 0.9999311475113775, + "pass@4": 0.9999994467434691, + "pass@8": 0.9999999999724691, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9964599609375, + "pass@2": 0.9990641916830709, + "pass@4": 0.9998087276199851, + "pass@8": 0.9999903205715637, + "pass@16": 0.9999999836447685, + "pass@32": 0.9999999999999961, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9976934523809524, + "pass@2": 0.9997941663542057, + "pass@4": 0.9999945093172878, + "pass@8": 0.9999999950689359, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9965615643712575, + "pass@2": 0.9996379043094914, + "pass@4": 0.9999889290597658, + "pass@8": 0.9999999886950237, + "pass@16": 0.9999999999999977, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9929271449704142, + "pass@2": 0.9980678959138986, + "pass@4": 0.9994872563081982, + "pass@8": 0.9999599346644329, + "pass@16": 0.9999998229513717, + "pass@32": 0.9999999999993783, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.985526842948718, + "pass@2": 0.9970164862204728, + "pass@4": 0.9995499270523874, + "pass@8": 0.9999784033248174, + "pass@16": 0.9999999467420494, + "pass@32": 0.9999999999999449, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9843970070422535, + "pass@2": 0.9951934540312746, + "pass@4": 0.9971520704278161, + "pass@8": 0.9975209774399802, + "pass@16": 0.9978457333734444, + "pass@32": 0.9984196517688809, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385446, + "answer_accuracy": 0.9866634583879423, + "avg_response_len": 198.4628624672346, + "resp_tokens_sum": 77530708, + "pass_at_k": { + "pass@1": 0.9866634583879423, + "pass@2": 0.9919017299228088, + "pass@4": 0.9941759165036379, + "pass@8": 0.9954912948932406, + "pass@16": 0.9963815420801608, + "pass@32": 0.9971040429636258, + "pass@64": 0.9976833659863941, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9227137653374233, + "pass@2": 0.9429375392493117, + "pass@4": 0.9563598661555336, + "pass@8": 0.9656292113866888, + "pass@16": 0.972287380308588, + "pass@32": 0.9775241038638645, + "pass@64": 0.981763116867637, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 0.9997862841945289, + "pass@2": 0.9999962604408492, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969223484848485, + "pass@2": 0.9985836465044142, + "pass@4": 0.9996747594050742, + "pass@8": 0.999983763588421, + "pass@16": 0.999999969633645, + "pass@32": 0.9999999999999856, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9955159023668639, + "pass@2": 0.9979375844476541, + "pass@4": 0.9990079075729438, + "pass@8": 0.9997006903613214, + "pass@16": 0.9999748631075642, + "pass@32": 0.9999999112627457, + "pass@64": 0.9999999999999892, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9943161231884058, + "pass@2": 0.9977643929019744, + "pass@4": 0.99949508675546, + "pass@8": 0.9999704874534221, + "pass@16": 0.9999999192465391, + "pass@32": 0.9999999999999238, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982091690544412, + "pass@2": 0.9999407756695169, + "pass@4": 0.9999997013267322, + "pass@8": 0.9999999999939795, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967711343930635, + "pass@2": 0.999786650584862, + "pass@4": 0.9999965644648466, + "pass@8": 0.9999999985045215, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9876980633802817, + "pass@2": 0.9934928468448486, + "pass@4": 0.995072504317242, + "pass@8": 0.9960171016781834, + "pass@16": 0.9971824274666318, + "pass@32": 0.9985597501374172, + "pass@64": 0.9996474842017711, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9855211452095808, + "pass@2": 0.9946945430006132, + "pass@4": 0.9964640314421774, + "pass@8": 0.996930973138194, + "pass@16": 0.9970046492451531, + "pass@32": 0.9970059879036105, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378302, + "answer_accuracy": 0.9894490709742216, + "avg_response_len": 212.0323380482089, + "resp_tokens_sum": 81067596, + "pass_at_k": { + "pass@1": 0.9894490709742216, + "pass@2": 0.9939684498970615, + "pass@4": 0.99571733335392, + "pass@8": 0.9968402273206528, + "pass@16": 0.9977014801770125, + "pass@32": 0.9983499483358242, + "pass@64": 0.99876366723885, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9493470149253731, + "pass@2": 0.9677135973674934, + "pass@4": 0.9777967773617846, + "pass@8": 0.9838416838151819, + "pass@16": 0.9877038473718193, + "pass@32": 0.9910220252708253, + "pass@64": 0.9934738597354361, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99984375, + "pass@2": 0.9999992969628797, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992028061224489, + "pass@2": 0.9999081747434632, + "pass@4": 0.9999975800218851, + "pass@8": 0.9999999990001509, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9984237938596491, + "pass@2": 0.9998186904268548, + "pass@4": 0.9999913708703079, + "pass@8": 0.9999999838662624, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9917049632352941, + "pass@2": 0.9958824253126447, + "pass@4": 0.9976440308012968, + "pass@8": 0.9987293865869793, + "pass@16": 0.9995273054917665, + "pass@32": 0.9999439253027383, + "pass@64": 0.9999998174959024, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992606466876972, + "pass@2": 0.9999922377605008, + "pass@4": 0.9999999982257738, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.994832871835443, + "pass@2": 0.9967256491079439, + "pass@4": 0.9968332677758637, + "pass@8": 0.9968354423301838, + "pass@16": 0.9968354430379747, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9857826576576577, + "pass@2": 0.9921396544181976, + "pass@4": 0.9939751430282925, + "pass@8": 0.9959554355169689, + "pass@16": 0.9981280304181758, + "pass@32": 0.9995587097302534, + "pass@64": 0.9999801661425524, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.993402188773831, + "pass@4": 0.995040290582648, + "pass@8": 0.995992442271155, + "pass@16": 0.9969055126132675, + "pass@32": 0.9975721294864157, + "pass@64": 0.9983922757459707, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.84428203125, + "avg_response_len": 382.38321875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.84428203125, + "pass@2": 0.9005478469487874, + "pass@4": 0.9337899588676353, + "pass@8": 0.9546169247362697, + "pass@16": 0.9682500787126251, + "pass@32": 0.9774101055822116, + "pass@64": 0.9834295612183087, + "pass@128": 0.9876 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9549453125, + "pass@2": 0.9763980068897639, + "pass@4": 0.987390393513311, + "pass@8": 0.9937273288594021, + "pass@16": 0.9970751544835733, + "pass@32": 0.9985467905245189, + "pass@64": 0.9989664644150142, + "pass@128": 0.999 + }, + "12": { + "pass@1": 0.9305390625, + "pass@2": 0.9597434793307087, + "pass@4": 0.9777261686351711, + "pass@8": 0.9885593337356346, + "pass@16": 0.9946306169225106, + "pass@32": 0.9975487461077769, + "pass@64": 0.9986899721242283, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.9095703125, + "pass@2": 0.9447161663385838, + "pass@4": 0.9659078782339704, + "pass@8": 0.9790371384652656, + "pass@16": 0.9875461788979727, + "pass@32": 0.9929803617992963, + "pass@64": 0.9962347043425379, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.909890625, + "pass@2": 0.9504397145669294, + "pass@4": 0.9719033106486681, + "pass@8": 0.9844675421856617, + "pass@16": 0.9922620675667974, + "pass@32": 0.9966419055379548, + "pass@64": 0.9988696740737704, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9070234375, + "pass@2": 0.9545851377952776, + "pass@4": 0.9755676227034124, + "pass@8": 0.9854491580530508, + "pass@16": 0.9906662263289284, + "pass@32": 0.9937758381778321, + "pass@64": 0.9952026948799971, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.8640703125, + "pass@2": 0.9132555364173235, + "pass@4": 0.9426584149793779, + "pass@8": 0.9610369098168507, + "pass@16": 0.9720578280473264, + "pass@32": 0.9791314734158106, + "pass@64": 0.9846902693289835, + "pass@128": 0.99 + }, + "17": { + "pass@1": 0.8513125, + "pass@2": 0.9044313484251991, + "pass@4": 0.9336355010311205, + "pass@8": 0.9527193105658855, + "pass@16": 0.9657324417989133, + "pass@32": 0.9746348160743804, + "pass@64": 0.980880900890739, + "pass@128": 0.987 + }, + "18": { + "pass@1": 0.8040078125, + "pass@2": 0.8820356791338599, + "pass@4": 0.9263771040494934, + "pass@8": 0.9523472675255719, + "pass@16": 0.9680954947186728, + "pass@32": 0.9791023069196815, + "pass@64": 0.9862262839085573, + "pass@128": 0.99 + }, + "19": { + "pass@1": 0.6722265625, + "pass@2": 0.7710553641732288, + "pass@4": 0.8361308353955762, + "pass@8": 0.8801712531053155, + "pass@16": 0.9111319472573388, + "pass@32": 0.9334601072151397, + "pass@64": 0.9487762803292228, + "pass@128": 0.959 + }, + "20": { + "pass@1": 0.639234375, + "pass@2": 0.7488180364173241, + "pass@4": 0.8206023594863132, + "pass@8": 0.8686540050500632, + "pass@16": 0.9033028311041832, + "pass@32": 0.9282787100496847, + "pass@64": 0.9457583678900183, + "pass@128": 0.958 + } + }, + "per_op_accuracy": { + "11": 0.9549453125, + "12": 0.9305390625, + "13": 0.9095703125, + "14": 0.909890625, + "15": 0.9070234375, + "16": 0.8640703125, + "17": 0.8513125, + "18": 0.8040078125, + "19": 0.6722265625, + "20": 0.639234375 + }, + "per_op_avg_response_len": { + "11": 301.289078125, + "12": 313.469484375, + "13": 334.3347578125, + "14": 361.40715625, + "15": 381.5985078125, + "16": 392.9100703125, + "17": 414.372703125, + "18": 432.0689921875, + "19": 436.1432890625, + "20": 456.2381484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 301.289078125, + "12": 313.469484375, + "13": 334.3347578125, + "14": 361.40715625, + "15": 381.5985078125, + "16": 392.9100703125, + "17": 414.372703125, + "18": 432.0689921875, + "19": 436.1432890625, + "20": 456.2381484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 358294, + "answer_accuracy": 0.8318489970282318, + "avg_response_len": 364.46228408246657, + "resp_tokens_sum": 156981195, + "pass_at_k": { + "pass@1": 0.8318489970282318, + "pass@2": 0.8929360543342254, + "pass@4": 0.9289027870773273, + "pass@8": 0.9512830734574134, + "pass@16": 0.9658239856328922, + "pass@32": 0.9756699617992833, + "pass@64": 0.9820331527602311, + "pass@128": 0.9860326894502229 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9420148725212465, + "pass@2": 0.9688956860208338, + "pass@4": 0.9842888575719818, + "pass@8": 0.9932980749767641, + "pass@16": 0.9975718340081712, + "pass@32": 0.999283473869785, + "pass@64": 0.999918274622942, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9253840042372882, + "pass@2": 0.9572093676320124, + "pass@4": 0.975868617588055, + "pass@8": 0.9874239409259805, + "pass@16": 0.9940122193279375, + "pass@32": 0.9967254534090262, + "pass@64": 0.9971693101066742, + "pass@128": 0.9971751412429378 + }, + "13": { + "pass@1": 0.9065683777429467, + "pass@2": 0.9460485677436875, + "pass@4": 0.969080333414436, + "pass@8": 0.9827606655847345, + "pass@16": 0.9899047973230172, + "pass@32": 0.9936117638775492, + "pass@64": 0.9964728083110211, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.8823014018691588, + "pass@2": 0.9289659835160793, + "pass@4": 0.956458548697768, + "pass@8": 0.9735331021115504, + "pass@16": 0.9854037725268124, + "pass@32": 0.9928036055443255, + "pass@64": 0.9970896386492748, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9151463150289018, + "pass@2": 0.9623722748168043, + "pass@4": 0.9810567303545147, + "pass@8": 0.9888010189100436, + "pass@16": 0.993003628745617, + "pass@32": 0.9959876521944565, + "pass@64": 0.9970619927265407, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8313508064516129, + "pass@2": 0.8921712280924569, + "pass@4": 0.9304420479698096, + "pass@8": 0.9540151944897635, + "pass@16": 0.9673493337288324, + "pass@32": 0.9748748941915307, + "pass@64": 0.9808457757289828, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.852194725433526, + "pass@2": 0.9099348999817946, + "pass@4": 0.9404819071568351, + "pass@8": 0.9589799397673455, + "pass@16": 0.970521670250947, + "pass@32": 0.9781434694857413, + "pass@64": 0.9817295210764027, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.75921875, + "pass@2": 0.8474205568053995, + "pass@4": 0.9002597238738005, + "pass@8": 0.9323657016061904, + "pass@16": 0.9531506347489849, + "pass@32": 0.9694149203288003, + "pass@64": 0.980450229236788, + "pass@128": 0.9857142857142858 + }, + "19": { + "pass@1": 0.6614583333333334, + "pass@2": 0.7607140851481399, + "pass@4": 0.8249680473499366, + "pass@8": 0.8700913512642859, + "pass@16": 0.9018850425263412, + "pass@32": 0.9242366811410543, + "pass@64": 0.9397228170742807, + "pass@128": 0.948948948948949 + }, + "20": { + "pass@1": 0.6349239864864865, + "pass@2": 0.7494402621125058, + "pass@4": 0.8215973997056667, + "pass@8": 0.8682736666753487, + "pass@16": 0.9029783755206716, + "pass@32": 0.9296180551931462, + "pass@64": 0.9484160357071253, + "pass@128": 0.960960960960961 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 368599, + "answer_accuracy": 0.8562829876598276, + "avg_response_len": 377.9988407857568, + "resp_tokens_sum": 162714893, + "pass_at_k": { + "pass@1": 0.8562829876598276, + "pass@2": 0.9085848546362629, + "pass@4": 0.9388879864314503, + "pass@8": 0.9578907518922528, + "pass@16": 0.97044611037026, + "pass@32": 0.9787367736563998, + "pass@64": 0.9841429346305324, + "pass@128": 0.9881058578650015 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9655967620481928, + "pass@2": 0.9813555402713219, + "pass@4": 0.9873279309589311, + "pass@8": 0.9915507045621954, + "pass@16": 0.9949525726882045, + "pass@32": 0.9966670644801715, + "pass@64": 0.9969831607359049, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9296385188087775, + "pass@2": 0.9592457205094661, + "pass@4": 0.9766451186156586, + "pass@8": 0.9861626009880742, + "pass@16": 0.9922937608317366, + "pass@32": 0.9965541033737507, + "pass@64": 0.9990781258535262, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9335828081232493, + "pass@2": 0.9606981572156414, + "pass@4": 0.9748942135909476, + "pass@8": 0.9834503879841167, + "pass@16": 0.9898332381537198, + "pass@32": 0.9943021535783841, + "pass@64": 0.9967150589030925, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.9181514985380117, + "pass@2": 0.9576616964820192, + "pass@4": 0.9784569757727651, + "pass@8": 0.9903096357097452, + "pass@16": 0.9965760665784712, + "pass@32": 0.9992521427633804, + "pass@64": 0.9999580995951128, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9094933712121213, + "pass@2": 0.9588310516583153, + "pass@4": 0.9822578816852432, + "pass@8": 0.9930328503480245, + "pass@16": 0.9969244873195249, + "pass@32": 0.9982614899741054, + "pass@64": 0.9992483524249482, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8821225071225072, + "pass@2": 0.9243023307983942, + "pass@4": 0.9472836953073169, + "pass@8": 0.9623024874009348, + "pass@16": 0.972876623671631, + "pass@32": 0.9807417394196154, + "pass@64": 0.9869335849984805, + "pass@128": 0.9914529914529915 + }, + "17": { + "pass@1": 0.8615100931677019, + "pass@2": 0.9146566427837828, + "pass@4": 0.943564751611017, + "pass@8": 0.9607675115201083, + "pass@16": 0.9711962890644932, + "pass@32": 0.9778534123969457, + "pass@64": 0.9834730810425003, + "pass@128": 0.9906832298136646 + }, + "18": { + "pass@1": 0.8249770220588235, + "pass@2": 0.8977278977535897, + "pass@4": 0.9374260079622395, + "pass@8": 0.9601756057206904, + "pass@16": 0.9739739781686679, + "pass@32": 0.983005959288786, + "pass@64": 0.98793702300017, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.6945917038690477, + "pass@2": 0.783559037542182, + "pass@4": 0.8407643528375911, + "pass@8": 0.8794797954531516, + "pass@16": 0.9074972703542947, + "pass@32": 0.9281647944725584, + "pass@64": 0.9420842485989787, + "pass@128": 0.9523809523809523 + }, + "20": { + "pass@1": 0.6413501122754491, + "pass@2": 0.746720879107926, + "pass@4": 0.8196913748431142, + "pass@8": 0.8713387468305476, + "pass@16": 0.9080377494829686, + "pass@32": 0.9322353462567267, + "pass@64": 0.9487201780875976, + "pass@128": 0.9610778443113772 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 353788, + "answer_accuracy": 0.8447337255501223, + "avg_response_len": 405.3198349633252, + "resp_tokens_sum": 169754432, + "pass_at_k": { + "pass@1": 0.8447337255501223, + "pass@2": 0.9001154585916341, + "pass@4": 0.9335762260978883, + "pass@8": 0.9546806560406009, + "pass@16": 0.9684880214842245, + "pass@32": 0.9778361444257387, + "pass@64": 0.9841324462048894, + "pass@128": 0.9886919315403423 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9582093253968254, + "pass@2": 0.979580286839145, + "pass@4": 0.9909319165461457, + "pass@8": 0.9965024586601782, + "pass@16": 0.9987556918927141, + "pass@32": 0.9997024090192648, + "pass@64": 0.999990171773953, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9369982798165137, + "pass@2": 0.9629724108454334, + "pass@4": 0.9807916977694299, + "pass@8": 0.9921265716594561, + "pass@16": 0.9975797601684947, + "pass@32": 0.9994103260695886, + "pass@64": 0.9999575357773413, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.8860677083333334, + "pass@2": 0.9257945404393898, + "pass@4": 0.952882770440732, + "pass@8": 0.9705083260290297, + "pass@16": 0.9827039584291745, + "pass@32": 0.9909022848607238, + "pass@64": 0.9954709952559196, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9277865356083086, + "pass@2": 0.9635647883128114, + "pass@4": 0.9799638896213635, + "pass@8": 0.9889540682347803, + "pass@16": 0.994416741293335, + "pass@32": 0.9976490069232948, + "pass@64": 0.9994606231626841, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.8958333333333334, + "pass@2": 0.9419447026586953, + "pass@4": 0.9628916451994424, + "pass@8": 0.9741455706645901, + "pass@16": 0.9817959566898227, + "pass@32": 0.9868451198367149, + "pass@64": 0.9890965712851263, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.8752995943952803, + "pass@2": 0.9210983409983046, + "pass@4": 0.9490407169789616, + "pass@8": 0.9661475706409914, + "pass@16": 0.9755157512762427, + "pass@32": 0.9813566539237482, + "pass@64": 0.9858831578717764, + "pass@128": 0.9911504424778761 + }, + "17": { + "pass@1": 0.8405026355421686, + "pass@2": 0.8887784158286688, + "pass@4": 0.916870214265385, + "pass@8": 0.9383888936653869, + "pass@16": 0.9554419843774683, + "pass@32": 0.9678565567484874, + "pass@64": 0.977482393080175, + "pass@128": 0.9879518072289156 + }, + "18": { + "pass@1": 0.8315776209677419, + "pass@2": 0.903906448437897, + "pass@4": 0.9437463160854891, + "pass@8": 0.9663211807044224, + "pass@16": 0.9785213547715544, + "pass@32": 0.98575825369811, + "pass@64": 0.9908713414697528, + "pass@128": 0.9935483870967742 + }, + "19": { + "pass@1": 0.6603568731117825, + "pass@2": 0.7687665776577776, + "pass@4": 0.8426575621589597, + "pass@8": 0.8910139633294577, + "pass@16": 0.9241243059124612, + "pass@32": 0.948114566321418, + "pass@64": 0.964677567112477, + "pass@128": 0.9758308157099698 + }, + "20": { + "pass@1": 0.6414226726726727, + "pass@2": 0.7502992658012341, + "pass@4": 0.8205210395997796, + "pass@8": 0.8663415393566618, + "pass@16": 0.8988781493348023, + "pass@32": 0.9229708469087713, + "pass@64": 0.9401299955504153, + "pass@128": 0.9519519519519519 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9121537828947368, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9121537828947368, + "pass@2": 0.944292782583905, + "pass@4": 0.9627557963017683, + "pass@8": 0.9742817180402421, + "pass@16": 0.9818740671993457, + "pass@32": 0.9870474767600861, + "pass@64": 0.9905071610179144, + "pass@128": 0.9928947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.934125, + "pass@2": 0.9531967273622055, + "pass@4": 0.9647547664041997, + "pass@8": 0.9726677273810179, + "pass@16": 0.9786270231393196, + "pass@32": 0.9837944277629593, + "pass@64": 0.988220290730658, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.99978125, + "pass@2": 0.999998031496063, + "pass@4": 0.9999999999062618, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982890625, + "pass@2": 0.999478592519685, + "pass@4": 0.9998916596362954, + "pass@8": 0.9999946416322282, + "pass@16": 0.9999999899791028, + "pass@32": 0.9999999999999952, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968125, + "pass@2": 0.9989414370078745, + "pass@4": 0.9996005144356952, + "pass@8": 0.9998957304072887, + "pass@16": 0.9999914984966795, + "pass@32": 0.9999999700068067, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9944921875, + "pass@2": 0.9977639025590551, + "pass@4": 0.9990230458380202, + "pass@8": 0.9995578080577183, + "pass@16": 0.9998392560072565, + "pass@32": 0.9999809346029047, + "pass@64": 0.9999999379486068, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9979921875, + "pass@2": 0.9998559301181105, + "pass@4": 0.9999961975065618, + "pass@8": 0.9999999962220367, + "pass@16": 0.9999999999999993, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.994859375, + "pass@2": 0.9982384350393703, + "pass@4": 0.9988248165541807, + "pass@8": 0.9989864571754808, + "pass@16": 0.9989999401575637, + "pass@32": 0.9989999999997898, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9863828125, + "pass@2": 0.9941416092519691, + "pass@4": 0.996104038901387, + "pass@8": 0.9972324929602487, + "pass@16": 0.9983763792634263, + "pass@32": 0.9993417616389402, + "pass@64": 0.9998682522170986, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9853671875, + "pass@2": 0.994469734251969, + "pass@4": 0.9962655018747654, + "pass@8": 0.9968485415656794, + "pass@16": 0.9972724026181798, + "pass@32": 0.9976839086080339, + "pass@64": 0.9982519662609338, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9549453125, + "pass@2": 0.9763980068897639, + "pass@4": 0.987390393513311, + "pass@8": 0.9937273288594021, + "pass@16": 0.9970751544835733, + "pass@32": 0.9985467905245189, + "pass@64": 0.9989664644150142, + "pass@128": 0.999 + }, + "12": { + "pass@1": 0.9305390625, + "pass@2": 0.9597434793307087, + "pass@4": 0.9777261686351711, + "pass@8": 0.9885593337356346, + "pass@16": 0.9946306169225106, + "pass@32": 0.9975487461077769, + "pass@64": 0.9986899721242283, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.9095703125, + "pass@2": 0.9447161663385838, + "pass@4": 0.9659078782339704, + "pass@8": 0.9790371384652656, + "pass@16": 0.9875461788979727, + "pass@32": 0.9929803617992963, + "pass@64": 0.9962347043425379, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.909890625, + "pass@2": 0.9504397145669294, + "pass@4": 0.9719033106486681, + "pass@8": 0.9844675421856617, + "pass@16": 0.9922620675667974, + "pass@32": 0.9966419055379548, + "pass@64": 0.9988696740737704, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9070234375, + "pass@2": 0.9545851377952776, + "pass@4": 0.9755676227034124, + "pass@8": 0.9854491580530508, + "pass@16": 0.9906662263289284, + "pass@32": 0.9937758381778321, + "pass@64": 0.9952026948799971, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.8640703125, + "pass@2": 0.9132555364173235, + "pass@4": 0.9426584149793779, + "pass@8": 0.9610369098168507, + "pass@16": 0.9720578280473264, + "pass@32": 0.9791314734158106, + "pass@64": 0.9846902693289835, + "pass@128": 0.99 + }, + "17": { + "pass@1": 0.8513125, + "pass@2": 0.9044313484251991, + "pass@4": 0.9336355010311205, + "pass@8": 0.9527193105658855, + "pass@16": 0.9657324417989133, + "pass@32": 0.9746348160743804, + "pass@64": 0.980880900890739, + "pass@128": 0.987 + }, + "18": { + "pass@1": 0.8040078125, + "pass@2": 0.8820356791338599, + "pass@4": 0.9263771040494934, + "pass@8": 0.9523472675255719, + "pass@16": 0.9680954947186728, + "pass@32": 0.9791023069196815, + "pass@64": 0.9862262839085573, + "pass@128": 0.99 + }, + "19": { + "pass@1": 0.6722265625, + "pass@2": 0.7710553641732288, + "pass@4": 0.8361308353955762, + "pass@8": 0.8801712531053155, + "pass@16": 0.9111319472573388, + "pass@32": 0.9334601072151397, + "pass@64": 0.9487762803292228, + "pass@128": 0.959 + }, + "20": { + "pass@1": 0.639234375, + "pass@2": 0.7488180364173241, + "pass@4": 0.8206023594863132, + "pass@8": 0.8686540050500632, + "pass@16": 0.9033028311041832, + "pass@32": 0.9282787100496847, + "pass@64": 0.9457583678900183, + "pass@128": 0.958 + } + }, + "per_op_accuracy": { + "10": 0.934125, + "2": 0.99978125, + "3": 0.9982890625, + "4": 0.9968125, + "5": 0.9944921875, + "6": 0.9979921875, + "7": 0.994859375, + "8": 0.9863828125, + "9": 0.9853671875, + "11": 0.9549453125, + "12": 0.9305390625, + "13": 0.9095703125, + "14": 0.909890625, + "15": 0.9070234375, + "16": 0.8640703125, + "17": 0.8513125, + "18": 0.8040078125, + "19": 0.6722265625, + "20": 0.639234375 + }, + "per_op_avg_response_len": { + "10": 281.61803125, + "2": 115.681953125, + "3": 145.0216015625, + "4": 160.1766953125, + "5": 173.8655703125, + "6": 199.1715, + "7": 224.4758125, + "8": 244.33884375, + "9": 259.932859375, + "11": 301.289078125, + "12": 313.469484375, + "13": 334.3347578125, + "14": 361.40715625, + "15": 381.5985078125, + "16": 392.9100703125, + "17": 414.372703125, + "18": 432.0689921875, + "19": 436.1432890625, + "20": 456.2381484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.61803125, + "2": 115.681953125, + "3": 145.0216015625, + "4": 160.1766953125, + "5": 173.8655703125, + "6": 199.1715, + "7": 224.4758125, + "8": 244.33884375, + "9": 259.932859375, + "11": 301.289078125, + "12": 313.469484375, + "13": 334.3347578125, + "14": 361.40715625, + "15": 381.5985078125, + "16": 392.9100703125, + "17": 414.372703125, + "18": 432.0689921875, + "19": 436.1432890625, + "20": 456.2381484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 732223, + "answer_accuracy": 0.9042826726999684, + "avg_response_len": 283.21991829355045, + "resp_tokens_sum": 229331098, + "pass_at_k": { + "pass@1": 0.9042826726999684, + "pass@2": 0.9397006612193992, + "pass@4": 0.9598143926138624, + "pass@8": 0.9722494066653309, + "pass@16": 0.9804004763829958, + "pass@32": 0.9860413514495616, + "pass@64": 0.9898269521281325, + "pass@128": 0.992254189061018 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9300562315634219, + "pass@2": 0.9487169157782271, + "pass@4": 0.9599396154020569, + "pass@8": 0.9683942194420995, + "pass@16": 0.9757538297615362, + "pass@32": 0.9826819806419275, + "pass@64": 0.9892382643670711, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997079439252337, + "pass@2": 0.9999984668972454, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9987098623853211, + "pass@2": 0.9999311475113775, + "pass@4": 0.9999994467434691, + "pass@8": 0.9999999999724691, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9964599609375, + "pass@2": 0.9990641916830709, + "pass@4": 0.9998087276199851, + "pass@8": 0.9999903205715637, + "pass@16": 0.9999999836447685, + "pass@32": 0.9999999999999961, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9976934523809524, + "pass@2": 0.9997941663542057, + "pass@4": 0.9999945093172878, + "pass@8": 0.9999999950689359, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9965615643712575, + "pass@2": 0.9996379043094914, + "pass@4": 0.9999889290597658, + "pass@8": 0.9999999886950237, + "pass@16": 0.9999999999999977, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9929271449704142, + "pass@2": 0.9980678959138986, + "pass@4": 0.9994872563081982, + "pass@8": 0.9999599346644329, + "pass@16": 0.9999998229513717, + "pass@32": 0.9999999999993783, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.985526842948718, + "pass@2": 0.9970164862204728, + "pass@4": 0.9995499270523874, + "pass@8": 0.9999784033248174, + "pass@16": 0.9999999467420494, + "pass@32": 0.9999999999999449, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9843970070422535, + "pass@2": 0.9951934540312746, + "pass@4": 0.9971520704278161, + "pass@8": 0.9975209774399802, + "pass@16": 0.9978457333734444, + "pass@32": 0.9984196517688809, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9420148725212465, + "pass@2": 0.9688956860208338, + "pass@4": 0.9842888575719818, + "pass@8": 0.9932980749767641, + "pass@16": 0.9975718340081712, + "pass@32": 0.999283473869785, + "pass@64": 0.999918274622942, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9253840042372882, + "pass@2": 0.9572093676320124, + "pass@4": 0.975868617588055, + "pass@8": 0.9874239409259805, + "pass@16": 0.9940122193279375, + "pass@32": 0.9967254534090262, + "pass@64": 0.9971693101066742, + "pass@128": 0.9971751412429378 + }, + "13": { + "pass@1": 0.9065683777429467, + "pass@2": 0.9460485677436875, + "pass@4": 0.969080333414436, + "pass@8": 0.9827606655847345, + "pass@16": 0.9899047973230172, + "pass@32": 0.9936117638775492, + "pass@64": 0.9964728083110211, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.8823014018691588, + "pass@2": 0.9289659835160793, + "pass@4": 0.956458548697768, + "pass@8": 0.9735331021115504, + "pass@16": 0.9854037725268124, + "pass@32": 0.9928036055443255, + "pass@64": 0.9970896386492748, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9151463150289018, + "pass@2": 0.9623722748168043, + "pass@4": 0.9810567303545147, + "pass@8": 0.9888010189100436, + "pass@16": 0.993003628745617, + "pass@32": 0.9959876521944565, + "pass@64": 0.9970619927265407, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8313508064516129, + "pass@2": 0.8921712280924569, + "pass@4": 0.9304420479698096, + "pass@8": 0.9540151944897635, + "pass@16": 0.9673493337288324, + "pass@32": 0.9748748941915307, + "pass@64": 0.9808457757289828, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.852194725433526, + "pass@2": 0.9099348999817946, + "pass@4": 0.9404819071568351, + "pass@8": 0.9589799397673455, + "pass@16": 0.970521670250947, + "pass@32": 0.9781434694857413, + "pass@64": 0.9817295210764027, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.75921875, + "pass@2": 0.8474205568053995, + "pass@4": 0.9002597238738005, + "pass@8": 0.9323657016061904, + "pass@16": 0.9531506347489849, + "pass@32": 0.9694149203288003, + "pass@64": 0.980450229236788, + "pass@128": 0.9857142857142858 + }, + "19": { + "pass@1": 0.6614583333333334, + "pass@2": 0.7607140851481399, + "pass@4": 0.8249680473499366, + "pass@8": 0.8700913512642859, + "pass@16": 0.9018850425263412, + "pass@32": 0.9242366811410543, + "pass@64": 0.9397228170742807, + "pass@128": 0.948948948948949 + }, + "20": { + "pass@1": 0.6349239864864865, + "pass@2": 0.7494402621125058, + "pass@4": 0.8215973997056667, + "pass@8": 0.8682736666753487, + "pass@16": 0.9029783755206716, + "pass@32": 0.9296180551931462, + "pass@64": 0.9484160357071253, + "pass@128": 0.960960960960961 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 754045, + "answer_accuracy": 0.9183127922837101, + "avg_response_len": 292.58281493569757, + "resp_tokens_sum": 240245601, + "pass_at_k": { + "pass@1": 0.9183127922837101, + "pass@2": 0.9482236860274367, + "pass@4": 0.9651917685951739, + "pass@8": 0.9757795838858602, + "pass@16": 0.9827851497433899, + "pass@32": 0.987475184556737, + "pass@64": 0.9905849294081015, + "pass@128": 0.9928293063133281 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9227137653374233, + "pass@2": 0.9429375392493117, + "pass@4": 0.9563598661555336, + "pass@8": 0.9656292113866888, + "pass@16": 0.972287380308588, + "pass@32": 0.9775241038638645, + "pass@64": 0.981763116867637, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 0.9997862841945289, + "pass@2": 0.9999962604408492, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969223484848485, + "pass@2": 0.9985836465044142, + "pass@4": 0.9996747594050742, + "pass@8": 0.999983763588421, + "pass@16": 0.999999969633645, + "pass@32": 0.9999999999999856, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9955159023668639, + "pass@2": 0.9979375844476541, + "pass@4": 0.9990079075729438, + "pass@8": 0.9997006903613214, + "pass@16": 0.9999748631075642, + "pass@32": 0.9999999112627457, + "pass@64": 0.9999999999999892, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9943161231884058, + "pass@2": 0.9977643929019744, + "pass@4": 0.99949508675546, + "pass@8": 0.9999704874534221, + "pass@16": 0.9999999192465391, + "pass@32": 0.9999999999999238, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982091690544412, + "pass@2": 0.9999407756695169, + "pass@4": 0.9999997013267322, + "pass@8": 0.9999999999939795, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967711343930635, + "pass@2": 0.999786650584862, + "pass@4": 0.9999965644648466, + "pass@8": 0.9999999985045215, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9876980633802817, + "pass@2": 0.9934928468448486, + "pass@4": 0.995072504317242, + "pass@8": 0.9960171016781834, + "pass@16": 0.9971824274666318, + "pass@32": 0.9985597501374172, + "pass@64": 0.9996474842017711, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9855211452095808, + "pass@2": 0.9946945430006132, + "pass@4": 0.9964640314421774, + "pass@8": 0.996930973138194, + "pass@16": 0.9970046492451531, + "pass@32": 0.9970059879036105, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9655967620481928, + "pass@2": 0.9813555402713219, + "pass@4": 0.9873279309589311, + "pass@8": 0.9915507045621954, + "pass@16": 0.9949525726882045, + "pass@32": 0.9966670644801715, + "pass@64": 0.9969831607359049, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9296385188087775, + "pass@2": 0.9592457205094661, + "pass@4": 0.9766451186156586, + "pass@8": 0.9861626009880742, + "pass@16": 0.9922937608317366, + "pass@32": 0.9965541033737507, + "pass@64": 0.9990781258535262, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9335828081232493, + "pass@2": 0.9606981572156414, + "pass@4": 0.9748942135909476, + "pass@8": 0.9834503879841167, + "pass@16": 0.9898332381537198, + "pass@32": 0.9943021535783841, + "pass@64": 0.9967150589030925, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.9181514985380117, + "pass@2": 0.9576616964820192, + "pass@4": 0.9784569757727651, + "pass@8": 0.9903096357097452, + "pass@16": 0.9965760665784712, + "pass@32": 0.9992521427633804, + "pass@64": 0.9999580995951128, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9094933712121213, + "pass@2": 0.9588310516583153, + "pass@4": 0.9822578816852432, + "pass@8": 0.9930328503480245, + "pass@16": 0.9969244873195249, + "pass@32": 0.9982614899741054, + "pass@64": 0.9992483524249482, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8821225071225072, + "pass@2": 0.9243023307983942, + "pass@4": 0.9472836953073169, + "pass@8": 0.9623024874009348, + "pass@16": 0.972876623671631, + "pass@32": 0.9807417394196154, + "pass@64": 0.9869335849984805, + "pass@128": 0.9914529914529915 + }, + "17": { + "pass@1": 0.8615100931677019, + "pass@2": 0.9146566427837828, + "pass@4": 0.943564751611017, + "pass@8": 0.9607675115201083, + "pass@16": 0.9711962890644932, + "pass@32": 0.9778534123969457, + "pass@64": 0.9834730810425003, + "pass@128": 0.9906832298136646 + }, + "18": { + "pass@1": 0.8249770220588235, + "pass@2": 0.8977278977535897, + "pass@4": 0.9374260079622395, + "pass@8": 0.9601756057206904, + "pass@16": 0.9739739781686679, + "pass@32": 0.983005959288786, + "pass@64": 0.98793702300017, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.6945917038690477, + "pass@2": 0.783559037542182, + "pass@4": 0.8407643528375911, + "pass@8": 0.8794797954531516, + "pass@16": 0.9074972703542947, + "pass@32": 0.9281647944725584, + "pass@64": 0.9420842485989787, + "pass@128": 0.9523809523809523 + }, + "20": { + "pass@1": 0.6413501122754491, + "pass@2": 0.746720879107926, + "pass@4": 0.8196913748431142, + "pass@8": 0.8713387468305476, + "pass@16": 0.9080377494829686, + "pass@32": 0.9322353462567267, + "pass@64": 0.9487201780875976, + "pass@128": 0.9610778443113772 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 732090, + "answer_accuracy": 0.9137966328486978, + "avg_response_len": 313.07670454545456, + "resp_tokens_sum": 250822028, + "pass_at_k": { + "pass@1": 0.9137966328486978, + "pass@2": 0.9449051829931684, + "pass@4": 0.9632319997636081, + "pass@8": 0.9748005856481224, + "pass@16": 0.9824296417295298, + "pass@32": 0.9876260041923829, + "pass@64": 0.9911149445638051, + "pass@128": 0.9936092027480428 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9493470149253731, + "pass@2": 0.9677135973674934, + "pass@4": 0.9777967773617846, + "pass@8": 0.9838416838151819, + "pass@16": 0.9877038473718193, + "pass@32": 0.9910220252708253, + "pass@64": 0.9934738597354361, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99984375, + "pass@2": 0.9999992969628797, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992028061224489, + "pass@2": 0.9999081747434632, + "pass@4": 0.9999975800218851, + "pass@8": 0.9999999990001509, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9984237938596491, + "pass@2": 0.9998186904268548, + "pass@4": 0.9999913708703079, + "pass@8": 0.9999999838662624, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9917049632352941, + "pass@2": 0.9958824253126447, + "pass@4": 0.9976440308012968, + "pass@8": 0.9987293865869793, + "pass@16": 0.9995273054917665, + "pass@32": 0.9999439253027383, + "pass@64": 0.9999998174959024, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992606466876972, + "pass@2": 0.9999922377605008, + "pass@4": 0.9999999982257738, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.994832871835443, + "pass@2": 0.9967256491079439, + "pass@4": 0.9968332677758637, + "pass@8": 0.9968354423301838, + "pass@16": 0.9968354430379747, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9857826576576577, + "pass@2": 0.9921396544181976, + "pass@4": 0.9939751430282925, + "pass@8": 0.9959554355169689, + "pass@16": 0.9981280304181758, + "pass@32": 0.9995587097302534, + "pass@64": 0.9999801661425524, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.993402188773831, + "pass@4": 0.995040290582648, + "pass@8": 0.995992442271155, + "pass@16": 0.9969055126132675, + "pass@32": 0.9975721294864157, + "pass@64": 0.9983922757459707, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9582093253968254, + "pass@2": 0.979580286839145, + "pass@4": 0.9909319165461457, + "pass@8": 0.9965024586601782, + "pass@16": 0.9987556918927141, + "pass@32": 0.9997024090192648, + "pass@64": 0.999990171773953, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9369982798165137, + "pass@2": 0.9629724108454334, + "pass@4": 0.9807916977694299, + "pass@8": 0.9921265716594561, + "pass@16": 0.9975797601684947, + "pass@32": 0.9994103260695886, + "pass@64": 0.9999575357773413, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.8860677083333334, + "pass@2": 0.9257945404393898, + "pass@4": 0.952882770440732, + "pass@8": 0.9705083260290297, + "pass@16": 0.9827039584291745, + "pass@32": 0.9909022848607238, + "pass@64": 0.9954709952559196, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9277865356083086, + "pass@2": 0.9635647883128114, + "pass@4": 0.9799638896213635, + "pass@8": 0.9889540682347803, + "pass@16": 0.994416741293335, + "pass@32": 0.9976490069232948, + "pass@64": 0.9994606231626841, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.8958333333333334, + "pass@2": 0.9419447026586953, + "pass@4": 0.9628916451994424, + "pass@8": 0.9741455706645901, + "pass@16": 0.9817959566898227, + "pass@32": 0.9868451198367149, + "pass@64": 0.9890965712851263, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.8752995943952803, + "pass@2": 0.9210983409983046, + "pass@4": 0.9490407169789616, + "pass@8": 0.9661475706409914, + "pass@16": 0.9755157512762427, + "pass@32": 0.9813566539237482, + "pass@64": 0.9858831578717764, + "pass@128": 0.9911504424778761 + }, + "17": { + "pass@1": 0.8405026355421686, + "pass@2": 0.8887784158286688, + "pass@4": 0.916870214265385, + "pass@8": 0.9383888936653869, + "pass@16": 0.9554419843774683, + "pass@32": 0.9678565567484874, + "pass@64": 0.977482393080175, + "pass@128": 0.9879518072289156 + }, + "18": { + "pass@1": 0.8315776209677419, + "pass@2": 0.903906448437897, + "pass@4": 0.9437463160854891, + "pass@8": 0.9663211807044224, + "pass@16": 0.9785213547715544, + "pass@32": 0.98575825369811, + "pass@64": 0.9908713414697528, + "pass@128": 0.9935483870967742 + }, + "19": { + "pass@1": 0.6603568731117825, + "pass@2": 0.7687665776577776, + "pass@4": 0.8426575621589597, + "pass@8": 0.8910139633294577, + "pass@16": 0.9241243059124612, + "pass@32": 0.948114566321418, + "pass@64": 0.964677567112477, + "pass@128": 0.9758308157099698 + }, + "20": { + "pass@1": 0.6414226726726727, + "pass@2": 0.7502992658012341, + "pass@4": 0.8205210395997796, + "pass@8": 0.8663415393566618, + "pass@16": 0.8988781493348023, + "pass@32": 0.9229708469087713, + "pass@64": 0.9401299955504153, + "pass@128": 0.9519519519519519 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7ace727aa17ebabd76c33444c894d7781214cfc7 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65b6a94c08fd3420f8f2c75a2b1a94847ce027c7723a1c4e6d9afc8d8170e040 +size 2724228770 diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a431957e01ca32b672ba923a2d010712f8c5623a --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive", "results_dir": "results/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 951001, "value_mismatch": 15919, "dependency_mismatch": 192056, "answer_mismatch": 14323, "extra_nodes": 93195, "missing_nodes": 88034, "perfect_with_extra": 2362, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 268630, "dependency_mismatch": 104244, "extra_nodes": 82400, "missing_nodes": 80854, "value_mismatch": 5305, "answer_mismatch": 5079, "perfect_with_extra": 581}, "teachers_in_school": {"total": 390656, "perfect": 344329, "value_mismatch": 5843, "answer_mismatch": 5210, "dependency_mismatch": 44732, "extra_nodes": 5540, "perfect_with_extra": 748, "missing_nodes": 4106}, "movie_festival_awards": {"total": 382336, "perfect": 338042, "extra_nodes": 5255, "perfect_with_extra": 1033, "value_mismatch": 4771, "answer_mismatch": 4034, "dependency_mismatch": 43080, "missing_nodes": 3074}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 562661, "value_mismatch": 260064, "dependency_mismatch": 673872, "answer_mismatch": 199319, "extra_nodes": 178589, "missing_nodes": 236262, "perfect_with_extra": 468, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 148649, "extra_nodes": 144125, "missing_nodes": 159036, "perfect_with_extra": 94, "value_mismatch": 91839, "dependency_mismatch": 268748, "answer_mismatch": 72427}, "movie_festival_awards": {"total": 418816, "perfect": 203683, "value_mismatch": 85386, "dependency_mismatch": 199597, "answer_mismatch": 65027, "extra_nodes": 16083, "missing_nodes": 38750, "perfect_with_extra": 199}, "teachers_in_school": {"total": 430464, "perfect": 210329, "dependency_mismatch": 205527, "value_mismatch": 82839, "answer_mismatch": 61865, "missing_nodes": 38476, "extra_nodes": 18381, "perfect_with_extra": 175}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1513662, "value_mismatch": 275983, "dependency_mismatch": 865928, "answer_mismatch": 213642, "extra_nodes": 271784, "missing_nodes": 324296, "perfect_with_extra": 2830, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 417279, "dependency_mismatch": 372992, "extra_nodes": 226525, "missing_nodes": 239890, "value_mismatch": 97144, "answer_mismatch": 77506, "perfect_with_extra": 675}, "teachers_in_school": {"total": 821120, "perfect": 554658, "value_mismatch": 88682, "answer_mismatch": 67075, "dependency_mismatch": 250259, "extra_nodes": 23921, "perfect_with_extra": 923, "missing_nodes": 42582}, "movie_festival_awards": {"total": 801152, "perfect": 541725, "extra_nodes": 21338, "perfect_with_extra": 1232, "value_mismatch": 90157, "answer_mismatch": 69061, "dependency_mismatch": 242677, "missing_nodes": 41824}}}} diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9be18f47c8295397a87c544e32d4661bed68cf5f --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc322bb5a741d6c37786a0bdd07f39607d2745d6ce3798571666144a2d922b46 +size 373509382 diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d48784ffaae6884b794b36852e408e132a19f3b0 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d5357f9ba6a8ab25ff12bbe872246595e17af595c41e86777cacf4685eeec2 +size 423628542 diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/results.json b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..925d3a4f3463be01af6c11f34cebb6cc9e5f4357 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-39470_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9190031152647975, + "pass@2": 0.9470404984423676, + "pass@4": 0.9750778816199377, + "pass@8": 0.9875389408099688, + "pass@16": 0.9968847352024922, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.8532110091743119, + "pass@2": 0.908256880733945, + "pass@4": 0.9510703363914373, + "pass@8": 0.9847094801223242, + "pass@16": 0.9877675840978594, + "pass@32": 0.9908256880733946, + "pass@64": 0.9938837920489296, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.765625, + "pass@2": 0.8625, + "pass@4": 0.928125, + "pass@8": 0.9625, + "pass@16": 0.975, + "pass@32": 0.978125, + "pass@64": 0.99375, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.7587301587301587, + "pass@2": 0.8412698412698413, + "pass@4": 0.9047619047619048, + "pass@8": 0.946031746031746, + "pass@16": 0.9619047619047619, + "pass@32": 0.9746031746031746, + "pass@64": 0.9809523809523809, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.7994011976047904, + "pass@2": 0.8682634730538922, + "pass@4": 0.9161676646706587, + "pass@8": 0.9431137724550899, + "pass@16": 0.9700598802395209, + "pass@32": 0.9850299401197605, + "pass@64": 0.9880239520958084, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.6686390532544378, + "pass@2": 0.7662721893491125, + "pass@4": 0.863905325443787, + "pass@8": 0.9112426035502958, + "pass@16": 0.9526627218934911, + "pass@32": 0.9674556213017751, + "pass@64": 0.9822485207100592, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.6025641025641025, + "pass@2": 0.7051282051282052, + "pass@4": 0.8012820512820513, + "pass@8": 0.8878205128205128, + "pass@16": 0.9166666666666666, + "pass@32": 0.9391025641025641, + "pass@64": 0.9615384615384616, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.5746478873239437, + "pass@2": 0.6816901408450704, + "pass@4": 0.7774647887323943, + "pass@8": 0.8366197183098592, + "pass@16": 0.8873239436619719, + "pass@32": 0.9380281690140845, + "pass@64": 0.9633802816901409, + "pass@128": 0.9661971830985916 + }, + "10": { + "pass@1": 0.471976401179941, + "pass@2": 0.5545722713864307, + "pass@4": 0.6548672566371682, + "pass@8": 0.7404129793510325, + "pass@16": 0.7935103244837758, + "pass@32": 0.8436578171091446, + "pass@64": 0.8761061946902655, + "pass@128": 0.9026548672566371 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9190031152647975, + "3": 0.8532110091743119, + "4": 0.765625, + "5": 0.7587301587301587, + "6": 0.7994011976047904, + "7": 0.6686390532544378, + "8": 0.6025641025641025, + "9": 0.5746478873239437, + "10": 0.471976401179941 + }, + "pass@2": { + "2": 0.9470404984423676, + "3": 0.908256880733945, + "4": 0.8625, + "5": 0.8412698412698413, + "6": 0.8682634730538922, + "7": 0.7662721893491125, + "8": 0.7051282051282052, + "9": 0.6816901408450704, + "10": 0.5545722713864307 + }, + "pass@4": { + "2": 0.9750778816199377, + "3": 0.9510703363914373, + "4": 0.928125, + "5": 0.9047619047619048, + "6": 0.9161676646706587, + "7": 0.863905325443787, + "8": 0.8012820512820513, + "9": 0.7774647887323943, + "10": 0.6548672566371682 + }, + "pass@8": { + "2": 0.9875389408099688, + "3": 0.9847094801223242, + "4": 0.9625, + "5": 0.946031746031746, + "6": 0.9431137724550899, + "7": 0.9112426035502958, + "8": 0.8878205128205128, + "9": 0.8366197183098592, + "10": 0.7404129793510325 + }, + "pass@16": { + "2": 0.9968847352024922, + "3": 0.9877675840978594, + "4": 0.975, + "5": 0.9619047619047619, + "6": 0.9700598802395209, + "7": 0.9526627218934911, + "8": 0.9166666666666666, + "9": 0.8873239436619719, + "10": 0.7935103244837758 + }, + "pass@32": { + "2": 1.0, + "3": 0.9908256880733946, + "4": 0.978125, + "5": 0.9746031746031746, + "6": 0.9850299401197605, + "7": 0.9674556213017751, + "8": 0.9391025641025641, + "9": 0.9380281690140845, + "10": 0.8436578171091446 + }, + "pass@64": { + "2": 1.0, + "3": 0.9938837920489296, + "4": 0.99375, + "5": 0.9809523809523809, + "6": 0.9880239520958084, + "7": 0.9822485207100592, + "8": 0.9615384615384616, + "9": 0.9633802816901409, + "10": 0.8761061946902655 + }, + "pass@128": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.996875, + "5": 0.9904761904761905, + "6": 0.9910179640718563, + "7": 0.9911242603550295, + "8": 0.9743589743589743, + "9": 0.9661971830985916, + "10": 0.9026548672566371 + } + }, + "total_pass_at_k": { + "pass@1": 0.7102330293819655, + "pass@2": 0.7906112799729821, + "pass@4": 0.861870989530564, + "pass@8": 0.9094900371496116, + "pass@16": 0.9368456602499156, + "pass@32": 0.9567713610266801, + "pass@64": 0.9706180344478217, + "pass@128": 0.9783856805133401 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9941520467836257, + "pass@2": 0.9970760233918129, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9470588235294117, + "pass@2": 0.9735294117647059, + "pass@4": 0.9911764705882353, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9684542586750788, + "pass@4": 0.9842271293375394, + "pass@8": 0.9873817034700315, + "pass@16": 0.9968454258675079, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8449367088607594, + "pass@2": 0.9177215189873418, + "pass@4": 0.9493670886075949, + "pass@8": 0.9778481012658228, + "pass@16": 0.990506329113924, + "pass@32": 0.9936708860759493, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.8018018018018018, + "pass@2": 0.8708708708708709, + "pass@4": 0.933933933933934, + "pass@8": 0.972972972972973, + "pass@16": 0.984984984984985, + "pass@32": 0.996996996996997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7684887459807074, + "pass@2": 0.819935691318328, + "pass@4": 0.887459807073955, + "pass@8": 0.9485530546623794, + "pass@16": 0.9839228295819936, + "pass@32": 0.9903536977491961, + "pass@64": 0.9967845659163987, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6059701492537314, + "pass@2": 0.6805970149253732, + "pass@4": 0.7492537313432835, + "pass@8": 0.8507462686567164, + "pass@16": 0.9104477611940298, + "pass@32": 0.9522388059701492, + "pass@64": 0.9582089552238806, + "pass@128": 0.9671641791044776 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9941520467836257, + "5": 0.9470588235294117, + "6": 0.9621451104100947, + "7": 0.8449367088607594, + "8": 0.8018018018018018, + "9": 0.7684887459807074, + "10": 0.6059701492537314 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9735294117647059, + "6": 0.9684542586750788, + "7": 0.9177215189873418, + "8": 0.8708708708708709, + "9": 0.819935691318328, + "10": 0.6805970149253732 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 0.9842271293375394, + "7": 0.9493670886075949, + "8": 0.933933933933934, + "9": 0.887459807073955, + "10": 0.7492537313432835 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9873817034700315, + "7": 0.9778481012658228, + "8": 0.972972972972973, + "9": 0.9485530546623794, + "10": 0.8507462686567164 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.984984984984985, + "9": 0.9839228295819936, + "10": 0.9104477611940298 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9903536977491961, + "10": 0.9522388059701492 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9582089552238806 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 1.0, + "10": 0.9671641791044776 + } + }, + "total_pass_at_k": { + "pass@1": 0.8821560093739538, + "pass@2": 0.9156344158018078, + "pass@4": 0.9444258453297623, + "pass@8": 0.970873786407767, + "pass@16": 0.9849347171074657, + "pass@32": 0.9922999665215936, + "pass@64": 0.9943086709072648, + "pass@128": 0.995647807164379 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9939393939393939, + "pass@2": 0.9939393939393939, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9733727810650887, + "pass@2": 0.9822485207100592, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9478260869565217, + "pass@2": 0.9884057971014493, + "pass@4": 0.9971014492753624, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9828080229226361, + "pass@2": 0.9856733524355301, + "pass@4": 0.9914040114613181, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8728323699421965, + "pass@2": 0.9219653179190751, + "pass@4": 0.9710982658959537, + "pass@8": 0.9884393063583815, + "pass@16": 0.9942196531791907, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8, + "pass@2": 0.8760563380281691, + "pass@4": 0.9267605633802817, + "pass@8": 0.9661971830985916, + "pass@16": 0.9859154929577465, + "pass@32": 0.9943661971830986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.718562874251497, + "pass@2": 0.8053892215568862, + "pass@4": 0.8682634730538922, + "pass@8": 0.9311377245508982, + "pass@16": 0.9670658682634731, + "pass@32": 0.9850299401197605, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "10": { + "pass@1": 0.6595092024539877, + "pass@2": 0.7208588957055214, + "pass@4": 0.7852760736196319, + "pass@8": 0.8404907975460123, + "pass@16": 0.8926380368098159, + "pass@32": 0.9202453987730062, + "pass@64": 0.950920245398773, + "pass@128": 0.9631901840490797 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9939393939393939, + "4": 0.9733727810650887, + "5": 0.9478260869565217, + "6": 0.9828080229226361, + "7": 0.8728323699421965, + "8": 0.8, + "9": 0.718562874251497, + "10": 0.6595092024539877 + }, + "pass@2": { + "2": 1.0, + "3": 0.9939393939393939, + "4": 0.9822485207100592, + "5": 0.9884057971014493, + "6": 0.9856733524355301, + "7": 0.9219653179190751, + "8": 0.8760563380281691, + "9": 0.8053892215568862, + "10": 0.7208588957055214 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.9710982658959537, + "8": 0.9267605633802817, + "9": 0.8682634730538922, + "10": 0.7852760736196319 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9884393063583815, + "8": 0.9661971830985916, + "9": 0.9311377245508982, + "10": 0.8404907975460123 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9942196531791907, + "8": 0.9859154929577465, + "9": 0.9670658682634731, + "10": 0.8926380368098159 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9943661971830986, + "9": 0.9850299401197605, + "10": 0.9202453987730062 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.950920245398773 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9631901840490797 + } + }, + "total_pass_at_k": { + "pass@1": 0.8836828309305373, + "pass@2": 0.9200524246395806, + "pass@4": 0.9495412844036697, + "pass@8": 0.9701834862385321, + "pass@16": 0.9826343381389253, + "pass@32": 0.9891874180865007, + "pass@64": 0.994429882044561, + "pass@128": 0.995740498034076 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.974, + "pass@2": 0.983, + "pass@4": 0.992, + "pass@8": 0.996, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.949, + "pass@2": 0.968, + "pass@4": 0.984, + "pass@8": 0.995, + "pass@16": 0.996, + "pass@32": 0.997, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.914, + "pass@2": 0.949, + "pass@4": 0.976, + "pass@8": 0.988, + "pass@16": 0.992, + "pass@32": 0.993, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.888, + "pass@2": 0.937, + "pass@4": 0.966, + "pass@8": 0.982, + "pass@16": 0.987, + "pass@32": 0.991, + "pass@64": 0.993, + "pass@128": 0.996 + }, + "6": { + "pass@1": 0.915, + "pass@2": 0.941, + "pass@4": 0.964, + "pass@8": 0.977, + "pass@16": 0.989, + "pass@32": 0.995, + "pass@64": 0.996, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.795, + "pass@2": 0.868, + "pass@4": 0.928, + "pass@8": 0.959, + "pass@16": 0.979, + "pass@32": 0.987, + "pass@64": 0.993, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.739, + "pass@2": 0.821, + "pass@4": 0.89, + "pass@8": 0.944, + "pass@16": 0.964, + "pass@32": 0.978, + "pass@64": 0.988, + "pass@128": 0.992 + }, + "9": { + "pass@1": 0.683, + "pass@2": 0.766, + "pass@4": 0.842, + "pass@8": 0.903, + "pass@16": 0.944, + "pass@32": 0.97, + "pass@64": 0.985, + "pass@128": 0.987 + }, + "10": { + "pass@1": 0.578, + "pass@2": 0.651, + "pass@4": 0.729, + "pass@8": 0.81, + "pass@16": 0.865, + "pass@32": 0.905, + "pass@64": 0.928, + "pass@128": 0.944 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.974, + "3": 0.949, + "4": 0.914, + "5": 0.888, + "6": 0.915, + "7": 0.795, + "8": 0.739, + "9": 0.683, + "10": 0.578 + }, + "pass@2": { + "2": 0.983, + "3": 0.968, + "4": 0.949, + "5": 0.937, + "6": 0.941, + "7": 0.868, + "8": 0.821, + "9": 0.766, + "10": 0.651 + }, + "pass@4": { + "2": 0.992, + "3": 0.984, + "4": 0.976, + "5": 0.966, + "6": 0.964, + "7": 0.928, + "8": 0.89, + "9": 0.842, + "10": 0.729 + }, + "pass@8": { + "2": 0.996, + "3": 0.995, + "4": 0.988, + "5": 0.982, + "6": 0.977, + "7": 0.959, + "8": 0.944, + "9": 0.903, + "10": 0.81 + }, + "pass@16": { + "2": 0.999, + "3": 0.996, + "4": 0.992, + "5": 0.987, + "6": 0.989, + "7": 0.979, + "8": 0.964, + "9": 0.944, + "10": 0.865 + }, + "pass@32": { + "2": 1.0, + "3": 0.997, + "4": 0.993, + "5": 0.991, + "6": 0.995, + "7": 0.987, + "8": 0.978, + "9": 0.97, + "10": 0.905 + }, + "pass@64": { + "2": 1.0, + "3": 0.998, + "4": 0.998, + "5": 0.993, + "6": 0.996, + "7": 0.993, + "8": 0.988, + "9": 0.985, + "10": 0.928 + }, + "pass@128": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.996, + "6": 0.997, + "7": 0.996, + "8": 0.992, + "9": 0.987, + "10": 0.944 + } + }, + "total_pass_at_k": { + "pass@1": 0.8261111111111111, + "pass@2": 0.876, + "pass@4": 0.919, + "pass@8": 0.9504444444444444, + "pass@16": 0.9683333333333334, + "pass@32": 0.9795555555555555, + "pass@64": 0.9865555555555555, + "pass@128": 0.99 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-39470_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.42209631728045327, + "pass@2": 0.5155807365439093, + "pass@4": 0.5892351274787535, + "pass@8": 0.6883852691218131, + "pass@16": 0.773371104815864, + "pass@32": 0.8215297450424929, + "pass@64": 0.8725212464589235, + "pass@128": 0.9178470254957507 + }, + "12": { + "pass@1": 0.4519774011299435, + "pass@2": 0.559322033898305, + "pass@4": 0.632768361581921, + "pass@8": 0.7288135593220338, + "pass@16": 0.788135593220339, + "pass@32": 0.844632768361582, + "pass@64": 0.8870056497175142, + "pass@128": 0.9124293785310734 + }, + "13": { + "pass@1": 0.43260188087774293, + "pass@2": 0.5172413793103449, + "pass@4": 0.5862068965517241, + "pass@8": 0.6520376175548589, + "pass@16": 0.7210031347962382, + "pass@32": 0.8025078369905956, + "pass@64": 0.8369905956112853, + "pass@128": 0.877742946708464 + }, + "14": { + "pass@1": 0.38317757009345793, + "pass@2": 0.4485981308411215, + "pass@4": 0.48286604361370716, + "pass@8": 0.5451713395638629, + "pass@16": 0.632398753894081, + "pass@32": 0.7071651090342679, + "pass@64": 0.8006230529595015, + "pass@128": 0.881619937694704 + }, + "15": { + "pass@1": 0.3670520231213873, + "pass@2": 0.4421965317919075, + "pass@4": 0.49421965317919075, + "pass@8": 0.5549132947976878, + "pass@16": 0.638728323699422, + "pass@32": 0.7254335260115607, + "pass@64": 0.8034682080924855, + "pass@128": 0.8670520231213873 + }, + "16": { + "pass@1": 0.32903225806451614, + "pass@2": 0.4032258064516129, + "pass@4": 0.4483870967741935, + "pass@8": 0.4967741935483871, + "pass@16": 0.5870967741935483, + "pass@32": 0.6580645161290323, + "pass@64": 0.7032258064516129, + "pass@128": 0.7354838709677419 + }, + "17": { + "pass@1": 0.3468208092485549, + "pass@2": 0.42485549132947975, + "pass@4": 0.47398843930635837, + "pass@8": 0.5317919075144508, + "pass@16": 0.6040462427745664, + "pass@32": 0.6647398843930635, + "pass@64": 0.7254335260115607, + "pass@128": 0.7716763005780347 + }, + "18": { + "pass@1": 0.2857142857142857, + "pass@2": 0.3657142857142857, + "pass@4": 0.44, + "pass@8": 0.4657142857142857, + "pass@16": 0.5342857142857143, + "pass@32": 0.58, + "pass@64": 0.6485714285714286, + "pass@128": 0.7028571428571428 + }, + "19": { + "pass@1": 0.24324324324324326, + "pass@2": 0.3063063063063063, + "pass@4": 0.34534534534534533, + "pass@8": 0.4084084084084084, + "pass@16": 0.4594594594594595, + "pass@32": 0.4984984984984985, + "pass@64": 0.5375375375375375, + "pass@128": 0.6006006006006006 + }, + "20": { + "pass@1": 0.1891891891891892, + "pass@2": 0.2672672672672673, + "pass@4": 0.33933933933933935, + "pass@8": 0.39039039039039036, + "pass@16": 0.44744744744744747, + "pass@32": 0.4984984984984985, + "pass@64": 0.5465465465465466, + "pass@128": 0.5915915915915916 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.42209631728045327, + "12": 0.4519774011299435, + "13": 0.43260188087774293, + "14": 0.38317757009345793, + "15": 0.3670520231213873, + "16": 0.32903225806451614, + "17": 0.3468208092485549, + "18": 0.2857142857142857, + "19": 0.24324324324324326, + "20": 0.1891891891891892 + }, + "pass@2": { + "11": 0.5155807365439093, + "12": 0.559322033898305, + "13": 0.5172413793103449, + "14": 0.4485981308411215, + "15": 0.4421965317919075, + "16": 0.4032258064516129, + "17": 0.42485549132947975, + "18": 0.3657142857142857, + "19": 0.3063063063063063, + "20": 0.2672672672672673 + }, + "pass@4": { + "11": 0.5892351274787535, + "12": 0.632768361581921, + "13": 0.5862068965517241, + "14": 0.48286604361370716, + "15": 0.49421965317919075, + "16": 0.4483870967741935, + "17": 0.47398843930635837, + "18": 0.44, + "19": 0.34534534534534533, + "20": 0.33933933933933935 + }, + "pass@8": { + "11": 0.6883852691218131, + "12": 0.7288135593220338, + "13": 0.6520376175548589, + "14": 0.5451713395638629, + "15": 0.5549132947976878, + "16": 0.4967741935483871, + "17": 0.5317919075144508, + "18": 0.4657142857142857, + "19": 0.4084084084084084, + "20": 0.39039039039039036 + }, + "pass@16": { + "11": 0.773371104815864, + "12": 0.788135593220339, + "13": 0.7210031347962382, + "14": 0.632398753894081, + "15": 0.638728323699422, + "16": 0.5870967741935483, + "17": 0.6040462427745664, + "18": 0.5342857142857143, + "19": 0.4594594594594595, + "20": 0.44744744744744747 + }, + "pass@32": { + "11": 0.8215297450424929, + "12": 0.844632768361582, + "13": 0.8025078369905956, + "14": 0.7071651090342679, + "15": 0.7254335260115607, + "16": 0.6580645161290323, + "17": 0.6647398843930635, + "18": 0.58, + "19": 0.4984984984984985, + "20": 0.4984984984984985 + }, + "pass@64": { + "11": 0.8725212464589235, + "12": 0.8870056497175142, + "13": 0.8369905956112853, + "14": 0.8006230529595015, + "15": 0.8034682080924855, + "16": 0.7032258064516129, + "17": 0.7254335260115607, + "18": 0.6485714285714286, + "19": 0.5375375375375375, + "20": 0.5465465465465466 + }, + "pass@128": { + "11": 0.9178470254957507, + "12": 0.9124293785310734, + "13": 0.877742946708464, + "14": 0.881619937694704, + "15": 0.8670520231213873, + "16": 0.7354838709677419, + "17": 0.7716763005780347, + "18": 0.7028571428571428, + "19": 0.6006006006006006, + "20": 0.5915915915915916 + } + }, + "total_pass_at_k": { + "pass@1": 0.34561664190193164, + "pass@2": 0.42585438335809805, + "pass@4": 0.48439821693907875, + "pass@8": 0.5476968796433879, + "pass@16": 0.6199108469539376, + "pass@32": 0.6811292719167905, + "pass@64": 0.737295690936107, + "pass@128": 0.7869242199108469 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.580952380952381, + "pass@2": 0.6571428571428571, + "pass@4": 0.7396825396825397, + "pass@8": 0.8444444444444444, + "pass@16": 0.9111111111111111, + "pass@32": 0.9587301587301588, + "pass@64": 0.9746031746031746, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.6207951070336392, + "pass@2": 0.6850152905198776, + "pass@4": 0.7614678899082569, + "pass@8": 0.8379204892966361, + "pass@16": 0.8990825688073395, + "pass@32": 0.9388379204892966, + "pass@64": 0.9724770642201835, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5648148148148148, + "pass@2": 0.6234567901234568, + "pass@4": 0.7191358024691358, + "pass@8": 0.7932098765432098, + "pass@16": 0.8703703703703703, + "pass@32": 0.9074074074074074, + "pass@64": 0.9259259259259259, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.516320474777448, + "pass@2": 0.5786350148367952, + "pass@4": 0.6350148367952523, + "pass@8": 0.6943620178041543, + "pass@16": 0.7655786350148368, + "pass@32": 0.8664688427299704, + "pass@64": 0.9258160237388724, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.4722222222222222, + "pass@2": 0.5, + "pass@4": 0.5555555555555556, + "pass@8": 0.6203703703703703, + "pass@16": 0.7283950617283951, + "pass@32": 0.8055555555555556, + "pass@64": 0.8672839506172839, + "pass@128": 0.9074074074074074 + }, + "16": { + "pass@1": 0.5162241887905604, + "pass@2": 0.5634218289085545, + "pass@4": 0.5958702064896755, + "pass@8": 0.6460176991150443, + "pass@16": 0.6991150442477876, + "pass@32": 0.7610619469026548, + "pass@64": 0.8082595870206489, + "pass@128": 0.8377581120943953 + }, + "17": { + "pass@1": 0.4819277108433735, + "pass@2": 0.5090361445783133, + "pass@4": 0.5421686746987951, + "pass@8": 0.5873493975903614, + "pass@16": 0.6536144578313253, + "pass@32": 0.7198795180722891, + "pass@64": 0.7801204819277109, + "pass@128": 0.8132530120481928 + }, + "18": { + "pass@1": 0.43548387096774194, + "pass@2": 0.5, + "pass@4": 0.5451612903225806, + "pass@8": 0.5870967741935483, + "pass@16": 0.635483870967742, + "pass@32": 0.7, + "pass@64": 0.7580645161290323, + "pass@128": 0.8225806451612904 + }, + "19": { + "pass@1": 0.3504531722054381, + "pass@2": 0.40483383685800606, + "pass@4": 0.43202416918429004, + "pass@8": 0.486404833836858, + "pass@16": 0.5287009063444109, + "pass@32": 0.5740181268882175, + "pass@64": 0.6193353474320241, + "pass@128": 0.6888217522658611 + }, + "20": { + "pass@1": 0.2912912912912913, + "pass@2": 0.35735735735735735, + "pass@4": 0.39039039039039036, + "pass@8": 0.42342342342342343, + "pass@16": 0.46846846846846846, + "pass@32": 0.5135135135135135, + "pass@64": 0.5615615615615616, + "pass@128": 0.6306306306306306 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.580952380952381, + "12": 0.6207951070336392, + "13": 0.5648148148148148, + "14": 0.516320474777448, + "15": 0.4722222222222222, + "16": 0.5162241887905604, + "17": 0.4819277108433735, + "18": 0.43548387096774194, + "19": 0.3504531722054381, + "20": 0.2912912912912913 + }, + "pass@2": { + "11": 0.6571428571428571, + "12": 0.6850152905198776, + "13": 0.6234567901234568, + "14": 0.5786350148367952, + "15": 0.5, + "16": 0.5634218289085545, + "17": 0.5090361445783133, + "18": 0.5, + "19": 0.40483383685800606, + "20": 0.35735735735735735 + }, + "pass@4": { + "11": 0.7396825396825397, + "12": 0.7614678899082569, + "13": 0.7191358024691358, + "14": 0.6350148367952523, + "15": 0.5555555555555556, + "16": 0.5958702064896755, + "17": 0.5421686746987951, + "18": 0.5451612903225806, + "19": 0.43202416918429004, + "20": 0.39039039039039036 + }, + "pass@8": { + "11": 0.8444444444444444, + "12": 0.8379204892966361, + "13": 0.7932098765432098, + "14": 0.6943620178041543, + "15": 0.6203703703703703, + "16": 0.6460176991150443, + "17": 0.5873493975903614, + "18": 0.5870967741935483, + "19": 0.486404833836858, + "20": 0.42342342342342343 + }, + "pass@16": { + "11": 0.9111111111111111, + "12": 0.8990825688073395, + "13": 0.8703703703703703, + "14": 0.7655786350148368, + "15": 0.7283950617283951, + "16": 0.6991150442477876, + "17": 0.6536144578313253, + "18": 0.635483870967742, + "19": 0.5287009063444109, + "20": 0.46846846846846846 + }, + "pass@32": { + "11": 0.9587301587301588, + "12": 0.9388379204892966, + "13": 0.9074074074074074, + "14": 0.8664688427299704, + "15": 0.8055555555555556, + "16": 0.7610619469026548, + "17": 0.7198795180722891, + "18": 0.7, + "19": 0.5740181268882175, + "20": 0.5135135135135135 + }, + "pass@64": { + "11": 0.9746031746031746, + "12": 0.9724770642201835, + "13": 0.9259259259259259, + "14": 0.9258160237388724, + "15": 0.8672839506172839, + "16": 0.8082595870206489, + "17": 0.7801204819277109, + "18": 0.7580645161290323, + "19": 0.6193353474320241, + "20": 0.5615615615615616 + }, + "pass@128": { + "11": 0.9936507936507937, + "12": 0.9755351681957186, + "13": 0.9567901234567902, + "14": 0.9465875370919882, + "15": 0.9074074074074074, + "16": 0.8377581120943953, + "17": 0.8132530120481928, + "18": 0.8225806451612904, + "19": 0.6888217522658611, + "20": 0.6306306306306306 + } + }, + "total_pass_at_k": { + "pass@1": 0.4825794621026895, + "pass@2": 0.5372860635696821, + "pass@4": 0.5907701711491442, + "pass@8": 0.6509779951100244, + "pass@16": 0.7148533007334963, + "pass@32": 0.7735330073349633, + "pass@64": 0.8184596577017115, + "pass@128": 0.8563569682151589 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6716867469879518, + "pass@2": 0.7198795180722891, + "pass@4": 0.8132530120481928, + "pass@8": 0.8765060240963856, + "pass@16": 0.9186746987951807, + "pass@32": 0.9518072289156626, + "pass@64": 0.9728915662650602, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.554858934169279, + "pass@2": 0.5987460815047022, + "pass@4": 0.7084639498432602, + "pass@8": 0.786833855799373, + "pass@16": 0.8746081504702194, + "pass@32": 0.9216300940438872, + "pass@64": 0.9561128526645768, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.6190476190476191, + "pass@2": 0.680672268907563, + "pass@4": 0.7647058823529411, + "pass@8": 0.8431372549019608, + "pass@16": 0.896358543417367, + "pass@32": 0.9355742296918768, + "pass@64": 0.9495798319327731, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.5263157894736842, + "pass@2": 0.5964912280701754, + "pass@4": 0.652046783625731, + "pass@8": 0.7076023391812866, + "pass@16": 0.7719298245614035, + "pass@32": 0.8421052631578947, + "pass@64": 0.9035087719298246, + "pass@128": 0.9532163742690059 + }, + "15": { + "pass@1": 0.5272727272727272, + "pass@2": 0.5727272727272728, + "pass@4": 0.6212121212121212, + "pass@8": 0.6818181818181818, + "pass@16": 0.7545454545454545, + "pass@32": 0.8272727272727273, + "pass@64": 0.9090909090909091, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5128205128205128, + "pass@2": 0.5441595441595442, + "pass@4": 0.5698005698005698, + "pass@8": 0.6267806267806267, + "pass@16": 0.6837606837606838, + "pass@32": 0.7578347578347578, + "pass@64": 0.8005698005698005, + "pass@128": 0.8490028490028491 + }, + "17": { + "pass@1": 0.4161490683229814, + "pass@2": 0.4503105590062112, + "pass@4": 0.4968944099378882, + "pass@8": 0.5434782608695652, + "pass@16": 0.6242236024844721, + "pass@32": 0.7049689440993789, + "pass@64": 0.7888198757763976, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.40294117647058825, + "pass@2": 0.45588235294117646, + "pass@4": 0.4764705882352941, + "pass@8": 0.5117647058823529, + "pass@16": 0.5705882352941176, + "pass@32": 0.6235294117647059, + "pass@64": 0.7029411764705882, + "pass@128": 0.7617647058823529 + }, + "19": { + "pass@1": 0.35119047619047616, + "pass@2": 0.3958333333333333, + "pass@4": 0.44047619047619047, + "pass@8": 0.4791666666666667, + "pass@16": 0.5238095238095238, + "pass@32": 0.6011904761904762, + "pass@64": 0.6607142857142857, + "pass@128": 0.7142857142857143 + }, + "20": { + "pass@1": 0.3203592814371258, + "pass@2": 0.36826347305389223, + "pass@4": 0.4161676646706587, + "pass@8": 0.45209580838323354, + "pass@16": 0.5029940119760479, + "pass@32": 0.5389221556886228, + "pass@64": 0.5718562874251497, + "pass@128": 0.5988023952095808 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6716867469879518, + "12": 0.554858934169279, + "13": 0.6190476190476191, + "14": 0.5263157894736842, + "15": 0.5272727272727272, + "16": 0.5128205128205128, + "17": 0.4161490683229814, + "18": 0.40294117647058825, + "19": 0.35119047619047616, + "20": 0.3203592814371258 + }, + "pass@2": { + "11": 0.7198795180722891, + "12": 0.5987460815047022, + "13": 0.680672268907563, + "14": 0.5964912280701754, + "15": 0.5727272727272728, + "16": 0.5441595441595442, + "17": 0.4503105590062112, + "18": 0.45588235294117646, + "19": 0.3958333333333333, + "20": 0.36826347305389223 + }, + "pass@4": { + "11": 0.8132530120481928, + "12": 0.7084639498432602, + "13": 0.7647058823529411, + "14": 0.652046783625731, + "15": 0.6212121212121212, + "16": 0.5698005698005698, + "17": 0.4968944099378882, + "18": 0.4764705882352941, + "19": 0.44047619047619047, + "20": 0.4161676646706587 + }, + "pass@8": { + "11": 0.8765060240963856, + "12": 0.786833855799373, + "13": 0.8431372549019608, + "14": 0.7076023391812866, + "15": 0.6818181818181818, + "16": 0.6267806267806267, + "17": 0.5434782608695652, + "18": 0.5117647058823529, + "19": 0.4791666666666667, + "20": 0.45209580838323354 + }, + "pass@16": { + "11": 0.9186746987951807, + "12": 0.8746081504702194, + "13": 0.896358543417367, + "14": 0.7719298245614035, + "15": 0.7545454545454545, + "16": 0.6837606837606838, + "17": 0.6242236024844721, + "18": 0.5705882352941176, + "19": 0.5238095238095238, + "20": 0.5029940119760479 + }, + "pass@32": { + "11": 0.9518072289156626, + "12": 0.9216300940438872, + "13": 0.9355742296918768, + "14": 0.8421052631578947, + "15": 0.8272727272727273, + "16": 0.7578347578347578, + "17": 0.7049689440993789, + "18": 0.6235294117647059, + "19": 0.6011904761904762, + "20": 0.5389221556886228 + }, + "pass@64": { + "11": 0.9728915662650602, + "12": 0.9561128526645768, + "13": 0.9495798319327731, + "14": 0.9035087719298246, + "15": 0.9090909090909091, + "16": 0.8005698005698005, + "17": 0.7888198757763976, + "18": 0.7029411764705882, + "19": 0.6607142857142857, + "20": 0.5718562874251497 + }, + "pass@128": { + "11": 0.9819277108433735, + "12": 0.9749216300940439, + "13": 0.969187675070028, + "14": 0.9532163742690059, + "15": 0.9424242424242424, + "16": 0.8490028490028491, + "17": 0.8322981366459627, + "18": 0.7617647058823529, + "19": 0.7142857142857143, + "20": 0.5988023952095808 + } + }, + "total_pass_at_k": { + "pass@1": 0.4909307166220636, + "pass@2": 0.5391019922688076, + "pass@4": 0.5964912280701754, + "pass@8": 0.6515016354445435, + "pass@16": 0.7124591138864109, + "pass@32": 0.7707404103479036, + "pass@64": 0.8215878679750223, + "pass@128": 0.8578650014867678 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.555, + "pass@2": 0.628, + "pass@4": 0.711, + "pass@8": 0.8, + "pass@16": 0.865, + "pass@32": 0.908, + "pass@64": 0.938, + "pass@128": 0.963 + }, + "12": { + "pass@1": 0.54, + "pass@2": 0.613, + "pass@4": 0.699, + "pass@8": 0.783, + "pass@16": 0.852, + "pass@32": 0.9, + "pass@64": 0.937, + "pass@128": 0.953 + }, + "13": { + "pass@1": 0.542, + "pass@2": 0.61, + "pass@4": 0.693, + "pass@8": 0.766, + "pass@16": 0.832, + "pass@32": 0.884, + "pass@64": 0.906, + "pass@128": 0.936 + }, + "14": { + "pass@1": 0.477, + "pass@2": 0.543, + "pass@4": 0.592, + "pass@8": 0.651, + "pass@16": 0.725, + "pass@32": 0.807, + "pass@64": 0.878, + "pass@128": 0.928 + }, + "15": { + "pass@1": 0.454, + "pass@2": 0.504, + "pass@4": 0.556, + "pass@8": 0.618, + "pass@16": 0.706, + "pass@32": 0.785, + "pass@64": 0.859, + "pass@128": 0.905 + }, + "16": { + "pass@1": 0.457, + "pass@2": 0.507, + "pass@4": 0.541, + "pass@8": 0.593, + "pass@16": 0.659, + "pass@32": 0.728, + "pass@64": 0.773, + "pass@128": 0.81 + }, + "17": { + "pass@1": 0.414, + "pass@2": 0.461, + "pass@4": 0.504, + "pass@8": 0.554, + "pass@16": 0.627, + "pass@32": 0.696, + "pass@64": 0.764, + "pass@128": 0.805 + }, + "18": { + "pass@1": 0.372, + "pass@2": 0.438, + "pass@4": 0.485, + "pass@8": 0.519, + "pass@16": 0.578, + "pass@32": 0.632, + "pass@64": 0.701, + "pass@128": 0.76 + }, + "19": { + "pass@1": 0.315, + "pass@2": 0.369, + "pass@4": 0.406, + "pass@8": 0.458, + "pass@16": 0.504, + "pass@32": 0.558, + "pass@64": 0.606, + "pass@128": 0.668 + }, + "20": { + "pass@1": 0.267, + "pass@2": 0.331, + "pass@4": 0.382, + "pass@8": 0.422, + "pass@16": 0.473, + "pass@32": 0.517, + "pass@64": 0.56, + "pass@128": 0.607 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.555, + "12": 0.54, + "13": 0.542, + "14": 0.477, + "15": 0.454, + "16": 0.457, + "17": 0.414, + "18": 0.372, + "19": 0.315, + "20": 0.267 + }, + "pass@2": { + "11": 0.628, + "12": 0.613, + "13": 0.61, + "14": 0.543, + "15": 0.504, + "16": 0.507, + "17": 0.461, + "18": 0.438, + "19": 0.369, + "20": 0.331 + }, + "pass@4": { + "11": 0.711, + "12": 0.699, + "13": 0.693, + "14": 0.592, + "15": 0.556, + "16": 0.541, + "17": 0.504, + "18": 0.485, + "19": 0.406, + "20": 0.382 + }, + "pass@8": { + "11": 0.8, + "12": 0.783, + "13": 0.766, + "14": 0.651, + "15": 0.618, + "16": 0.593, + "17": 0.554, + "18": 0.519, + "19": 0.458, + "20": 0.422 + }, + "pass@16": { + "11": 0.865, + "12": 0.852, + "13": 0.832, + "14": 0.725, + "15": 0.706, + "16": 0.659, + "17": 0.627, + "18": 0.578, + "19": 0.504, + "20": 0.473 + }, + "pass@32": { + "11": 0.908, + "12": 0.9, + "13": 0.884, + "14": 0.807, + "15": 0.785, + "16": 0.728, + "17": 0.696, + "18": 0.632, + "19": 0.558, + "20": 0.517 + }, + "pass@64": { + "11": 0.938, + "12": 0.937, + "13": 0.906, + "14": 0.878, + "15": 0.859, + "16": 0.773, + "17": 0.764, + "18": 0.701, + "19": 0.606, + "20": 0.56 + }, + "pass@128": { + "11": 0.963, + "12": 0.953, + "13": 0.936, + "14": 0.928, + "15": 0.905, + "16": 0.81, + "17": 0.805, + "18": 0.76, + "19": 0.668, + "20": 0.607 + } + }, + "total_pass_at_k": { + "pass@1": 0.4393, + "pass@2": 0.5004, + "pass@4": 0.5569, + "pass@8": 0.6164, + "pass@16": 0.6821, + "pass@32": 0.7415, + "pass@64": 0.7922, + "pass@128": 0.8335 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/summary.csv b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..424f35fb436caa1cd41c46ed7025d9cffaad8732 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-39470,0.9875668402777777,0.84428203125,0.9121537828947368,0.0,0.0,0.0,200.47587413194444,382.38321875,1152000,1280000,2432000 diff --git a/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/summary.json b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1e8fb6c80f1f55a49fdda5a152822629e9fb8e4c --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-39470", + "id_acc": 0.9875668402777777, + "ood_acc": 0.84428203125, + "total_acc": 0.9121537828947368, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.47587413194444, + "ood_avg_resp_len": 382.38321875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 732223, + "answer_accuracy": 0.9042826726999684, + "avg_response_len": 283.21991829355045, + "resp_tokens_sum": 229331098, + "pass_at_k": { + "pass@1": 0.9042826726999684, + "pass@2": 0.9397006612193992, + "pass@4": 0.9598143926138624, + "pass@8": 0.9722494066653309, + "pass@16": 0.9804004763829958, + "pass@32": 0.9860413514495616, + "pass@64": 0.9898269521281325, + "pass@128": 0.992254189061018 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9300562315634219, + "pass@2": 0.9487169157782271, + "pass@4": 0.9599396154020569, + "pass@8": 0.9683942194420995, + "pass@16": 0.9757538297615362, + "pass@32": 0.9826819806419275, + "pass@64": 0.9892382643670711, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997079439252337, + "pass@2": 0.9999984668972454, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9987098623853211, + "pass@2": 0.9999311475113775, + "pass@4": 0.9999994467434691, + "pass@8": 0.9999999999724691, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9964599609375, + "pass@2": 0.9990641916830709, + "pass@4": 0.9998087276199851, + "pass@8": 0.9999903205715637, + "pass@16": 0.9999999836447685, + "pass@32": 0.9999999999999961, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9976934523809524, + "pass@2": 0.9997941663542057, + "pass@4": 0.9999945093172878, + "pass@8": 0.9999999950689359, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9965615643712575, + "pass@2": 0.9996379043094914, + "pass@4": 0.9999889290597658, + "pass@8": 0.9999999886950237, + "pass@16": 0.9999999999999977, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9929271449704142, + "pass@2": 0.9980678959138986, + "pass@4": 0.9994872563081982, + "pass@8": 0.9999599346644329, + "pass@16": 0.9999998229513717, + "pass@32": 0.9999999999993783, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.985526842948718, + "pass@2": 0.9970164862204728, + "pass@4": 0.9995499270523874, + "pass@8": 0.9999784033248174, + "pass@16": 0.9999999467420494, + "pass@32": 0.9999999999999449, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9843970070422535, + "pass@2": 0.9951934540312746, + "pass@4": 0.9971520704278161, + "pass@8": 0.9975209774399802, + "pass@16": 0.9978457333734444, + "pass@32": 0.9984196517688809, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9420148725212465, + "pass@2": 0.9688956860208338, + "pass@4": 0.9842888575719818, + "pass@8": 0.9932980749767641, + "pass@16": 0.9975718340081712, + "pass@32": 0.999283473869785, + "pass@64": 0.999918274622942, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9253840042372882, + "pass@2": 0.9572093676320124, + "pass@4": 0.975868617588055, + "pass@8": 0.9874239409259805, + "pass@16": 0.9940122193279375, + "pass@32": 0.9967254534090262, + "pass@64": 0.9971693101066742, + "pass@128": 0.9971751412429378 + }, + "13": { + "pass@1": 0.9065683777429467, + "pass@2": 0.9460485677436875, + "pass@4": 0.969080333414436, + "pass@8": 0.9827606655847345, + "pass@16": 0.9899047973230172, + "pass@32": 0.9936117638775492, + "pass@64": 0.9964728083110211, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.8823014018691588, + "pass@2": 0.9289659835160793, + "pass@4": 0.956458548697768, + "pass@8": 0.9735331021115504, + "pass@16": 0.9854037725268124, + "pass@32": 0.9928036055443255, + "pass@64": 0.9970896386492748, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9151463150289018, + "pass@2": 0.9623722748168043, + "pass@4": 0.9810567303545147, + "pass@8": 0.9888010189100436, + "pass@16": 0.993003628745617, + "pass@32": 0.9959876521944565, + "pass@64": 0.9970619927265407, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8313508064516129, + "pass@2": 0.8921712280924569, + "pass@4": 0.9304420479698096, + "pass@8": 0.9540151944897635, + "pass@16": 0.9673493337288324, + "pass@32": 0.9748748941915307, + "pass@64": 0.9808457757289828, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.852194725433526, + "pass@2": 0.9099348999817946, + "pass@4": 0.9404819071568351, + "pass@8": 0.9589799397673455, + "pass@16": 0.970521670250947, + "pass@32": 0.9781434694857413, + "pass@64": 0.9817295210764027, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.75921875, + "pass@2": 0.8474205568053995, + "pass@4": 0.9002597238738005, + "pass@8": 0.9323657016061904, + "pass@16": 0.9531506347489849, + "pass@32": 0.9694149203288003, + "pass@64": 0.980450229236788, + "pass@128": 0.9857142857142858 + }, + "19": { + "pass@1": 0.6614583333333334, + "pass@2": 0.7607140851481399, + "pass@4": 0.8249680473499366, + "pass@8": 0.8700913512642859, + "pass@16": 0.9018850425263412, + "pass@32": 0.9242366811410543, + "pass@64": 0.9397228170742807, + "pass@128": 0.948948948948949 + }, + "20": { + "pass@1": 0.6349239864864865, + "pass@2": 0.7494402621125058, + "pass@4": 0.8215973997056667, + "pass@8": 0.8682736666753487, + "pass@16": 0.9029783755206716, + "pass@32": 0.9296180551931462, + "pass@64": 0.9484160357071253, + "pass@128": 0.960960960960961 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 754045, + "answer_accuracy": 0.9183127922837101, + "avg_response_len": 292.58281493569757, + "resp_tokens_sum": 240245601, + "pass_at_k": { + "pass@1": 0.9183127922837101, + "pass@2": 0.9482236860274367, + "pass@4": 0.9651917685951739, + "pass@8": 0.9757795838858602, + "pass@16": 0.9827851497433899, + "pass@32": 0.987475184556737, + "pass@64": 0.9905849294081015, + "pass@128": 0.9928293063133281 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9227137653374233, + "pass@2": 0.9429375392493117, + "pass@4": 0.9563598661555336, + "pass@8": 0.9656292113866888, + "pass@16": 0.972287380308588, + "pass@32": 0.9775241038638645, + "pass@64": 0.981763116867637, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 0.9997862841945289, + "pass@2": 0.9999962604408492, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969223484848485, + "pass@2": 0.9985836465044142, + "pass@4": 0.9996747594050742, + "pass@8": 0.999983763588421, + "pass@16": 0.999999969633645, + "pass@32": 0.9999999999999856, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9955159023668639, + "pass@2": 0.9979375844476541, + "pass@4": 0.9990079075729438, + "pass@8": 0.9997006903613214, + "pass@16": 0.9999748631075642, + "pass@32": 0.9999999112627457, + "pass@64": 0.9999999999999892, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9943161231884058, + "pass@2": 0.9977643929019744, + "pass@4": 0.99949508675546, + "pass@8": 0.9999704874534221, + "pass@16": 0.9999999192465391, + "pass@32": 0.9999999999999238, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982091690544412, + "pass@2": 0.9999407756695169, + "pass@4": 0.9999997013267322, + "pass@8": 0.9999999999939795, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967711343930635, + "pass@2": 0.999786650584862, + "pass@4": 0.9999965644648466, + "pass@8": 0.9999999985045215, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9876980633802817, + "pass@2": 0.9934928468448486, + "pass@4": 0.995072504317242, + "pass@8": 0.9960171016781834, + "pass@16": 0.9971824274666318, + "pass@32": 0.9985597501374172, + "pass@64": 0.9996474842017711, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9855211452095808, + "pass@2": 0.9946945430006132, + "pass@4": 0.9964640314421774, + "pass@8": 0.996930973138194, + "pass@16": 0.9970046492451531, + "pass@32": 0.9970059879036105, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9655967620481928, + "pass@2": 0.9813555402713219, + "pass@4": 0.9873279309589311, + "pass@8": 0.9915507045621954, + "pass@16": 0.9949525726882045, + "pass@32": 0.9966670644801715, + "pass@64": 0.9969831607359049, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.9296385188087775, + "pass@2": 0.9592457205094661, + "pass@4": 0.9766451186156586, + "pass@8": 0.9861626009880742, + "pass@16": 0.9922937608317366, + "pass@32": 0.9965541033737507, + "pass@64": 0.9990781258535262, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9335828081232493, + "pass@2": 0.9606981572156414, + "pass@4": 0.9748942135909476, + "pass@8": 0.9834503879841167, + "pass@16": 0.9898332381537198, + "pass@32": 0.9943021535783841, + "pass@64": 0.9967150589030925, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.9181514985380117, + "pass@2": 0.9576616964820192, + "pass@4": 0.9784569757727651, + "pass@8": 0.9903096357097452, + "pass@16": 0.9965760665784712, + "pass@32": 0.9992521427633804, + "pass@64": 0.9999580995951128, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9094933712121213, + "pass@2": 0.9588310516583153, + "pass@4": 0.9822578816852432, + "pass@8": 0.9930328503480245, + "pass@16": 0.9969244873195249, + "pass@32": 0.9982614899741054, + "pass@64": 0.9992483524249482, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8821225071225072, + "pass@2": 0.9243023307983942, + "pass@4": 0.9472836953073169, + "pass@8": 0.9623024874009348, + "pass@16": 0.972876623671631, + "pass@32": 0.9807417394196154, + "pass@64": 0.9869335849984805, + "pass@128": 0.9914529914529915 + }, + "17": { + "pass@1": 0.8615100931677019, + "pass@2": 0.9146566427837828, + "pass@4": 0.943564751611017, + "pass@8": 0.9607675115201083, + "pass@16": 0.9711962890644932, + "pass@32": 0.9778534123969457, + "pass@64": 0.9834730810425003, + "pass@128": 0.9906832298136646 + }, + "18": { + "pass@1": 0.8249770220588235, + "pass@2": 0.8977278977535897, + "pass@4": 0.9374260079622395, + "pass@8": 0.9601756057206904, + "pass@16": 0.9739739781686679, + "pass@32": 0.983005959288786, + "pass@64": 0.98793702300017, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.6945917038690477, + "pass@2": 0.783559037542182, + "pass@4": 0.8407643528375911, + "pass@8": 0.8794797954531516, + "pass@16": 0.9074972703542947, + "pass@32": 0.9281647944725584, + "pass@64": 0.9420842485989787, + "pass@128": 0.9523809523809523 + }, + "20": { + "pass@1": 0.6413501122754491, + "pass@2": 0.746720879107926, + "pass@4": 0.8196913748431142, + "pass@8": 0.8713387468305476, + "pass@16": 0.9080377494829686, + "pass@32": 0.9322353462567267, + "pass@64": 0.9487201780875976, + "pass@128": 0.9610778443113772 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 732090, + "answer_accuracy": 0.9137966328486978, + "avg_response_len": 313.07670454545456, + "resp_tokens_sum": 250822028, + "pass_at_k": { + "pass@1": 0.9137966328486978, + "pass@2": 0.9449051829931684, + "pass@4": 0.9632319997636081, + "pass@8": 0.9748005856481224, + "pass@16": 0.9824296417295298, + "pass@32": 0.9876260041923829, + "pass@64": 0.9911149445638051, + "pass@128": 0.9936092027480428 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9493470149253731, + "pass@2": 0.9677135973674934, + "pass@4": 0.9777967773617846, + "pass@8": 0.9838416838151819, + "pass@16": 0.9877038473718193, + "pass@32": 0.9910220252708253, + "pass@64": 0.9934738597354361, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99984375, + "pass@2": 0.9999992969628797, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992028061224489, + "pass@2": 0.9999081747434632, + "pass@4": 0.9999975800218851, + "pass@8": 0.9999999990001509, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9984237938596491, + "pass@2": 0.9998186904268548, + "pass@4": 0.9999913708703079, + "pass@8": 0.9999999838662624, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9917049632352941, + "pass@2": 0.9958824253126447, + "pass@4": 0.9976440308012968, + "pass@8": 0.9987293865869793, + "pass@16": 0.9995273054917665, + "pass@32": 0.9999439253027383, + "pass@64": 0.9999998174959024, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992606466876972, + "pass@2": 0.9999922377605008, + "pass@4": 0.9999999982257738, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.994832871835443, + "pass@2": 0.9967256491079439, + "pass@4": 0.9968332677758637, + "pass@8": 0.9968354423301838, + "pass@16": 0.9968354430379747, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9857826576576577, + "pass@2": 0.9921396544181976, + "pass@4": 0.9939751430282925, + "pass@8": 0.9959554355169689, + "pass@16": 0.9981280304181758, + "pass@32": 0.9995587097302534, + "pass@64": 0.9999801661425524, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.993402188773831, + "pass@4": 0.995040290582648, + "pass@8": 0.995992442271155, + "pass@16": 0.9969055126132675, + "pass@32": 0.9975721294864157, + "pass@64": 0.9983922757459707, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9582093253968254, + "pass@2": 0.979580286839145, + "pass@4": 0.9909319165461457, + "pass@8": 0.9965024586601782, + "pass@16": 0.9987556918927141, + "pass@32": 0.9997024090192648, + "pass@64": 0.999990171773953, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9369982798165137, + "pass@2": 0.9629724108454334, + "pass@4": 0.9807916977694299, + "pass@8": 0.9921265716594561, + "pass@16": 0.9975797601684947, + "pass@32": 0.9994103260695886, + "pass@64": 0.9999575357773413, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.8860677083333334, + "pass@2": 0.9257945404393898, + "pass@4": 0.952882770440732, + "pass@8": 0.9705083260290297, + "pass@16": 0.9827039584291745, + "pass@32": 0.9909022848607238, + "pass@64": 0.9954709952559196, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9277865356083086, + "pass@2": 0.9635647883128114, + "pass@4": 0.9799638896213635, + "pass@8": 0.9889540682347803, + "pass@16": 0.994416741293335, + "pass@32": 0.9976490069232948, + "pass@64": 0.9994606231626841, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.8958333333333334, + "pass@2": 0.9419447026586953, + "pass@4": 0.9628916451994424, + "pass@8": 0.9741455706645901, + "pass@16": 0.9817959566898227, + "pass@32": 0.9868451198367149, + "pass@64": 0.9890965712851263, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.8752995943952803, + "pass@2": 0.9210983409983046, + "pass@4": 0.9490407169789616, + "pass@8": 0.9661475706409914, + "pass@16": 0.9755157512762427, + "pass@32": 0.9813566539237482, + "pass@64": 0.9858831578717764, + "pass@128": 0.9911504424778761 + }, + "17": { + "pass@1": 0.8405026355421686, + "pass@2": 0.8887784158286688, + "pass@4": 0.916870214265385, + "pass@8": 0.9383888936653869, + "pass@16": 0.9554419843774683, + "pass@32": 0.9678565567484874, + "pass@64": 0.977482393080175, + "pass@128": 0.9879518072289156 + }, + "18": { + "pass@1": 0.8315776209677419, + "pass@2": 0.903906448437897, + "pass@4": 0.9437463160854891, + "pass@8": 0.9663211807044224, + "pass@16": 0.9785213547715544, + "pass@32": 0.98575825369811, + "pass@64": 0.9908713414697528, + "pass@128": 0.9935483870967742 + }, + "19": { + "pass@1": 0.6603568731117825, + "pass@2": 0.7687665776577776, + "pass@4": 0.8426575621589597, + "pass@8": 0.8910139633294577, + "pass@16": 0.9241243059124612, + "pass@32": 0.948114566321418, + "pass@64": 0.964677567112477, + "pass@128": 0.9758308157099698 + }, + "20": { + "pass@1": 0.6414226726726727, + "pass@2": 0.7502992658012341, + "pass@4": 0.8205210395997796, + "pass@8": 0.8663415393566618, + "pass@16": 0.8988781493348023, + "pass@32": 0.9229708469087713, + "pass@64": 0.9401299955504153, + "pass@128": 0.9519519519519519 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_id_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0f4f1cb0ca36e7ddf35769f9c682f2bff3e6cc97 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0ce7fd42f2782e5fddd9bc4add013ec410ac8fd5a267ea943b45bf122ca01c8 +size 1789699693 diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_metrics.json b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d8b5197ebb1035c13720629f4cf7f81a2128b114 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-39008", + "id": { + "answer_accuracy": 0.9902135416666666, + "avg_response_len": 200.16405208333333, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9902135416666666, + "pass@2": 0.9927832185039359, + "pass@4": 0.9945416849456318, + "pass@8": 0.9958928105928826, + "pass@16": 0.9969937087834242, + "pass@32": 0.997979060211912, + "pass@64": 0.9988172035093432, + "pass@128": 0.9993333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9376640625, + "pass@2": 0.9507861712598423, + "pass@4": 0.9622442902137232, + "pass@8": 0.9716912530130499, + "pass@16": 0.9792983621669582, + "pass@32": 0.9860150425510199, + "pass@64": 0.9919049625799984, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988671875, + "pass@2": 0.999171875, + "pass@4": 0.9993288038057743, + "pass@8": 0.9995556419393896, + "pass@16": 0.9998138373001255, + "pass@32": 0.9999736716068084, + "pass@64": 0.9999998615776614, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998046875, + "pass@2": 0.999976624015748, + "pass@4": 0.9999995458380202, + "pass@8": 0.9999999999118908, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9938828125, + "pass@2": 0.9960645915354331, + "pass@4": 0.9969954731908509, + "pass@8": 0.9975564679095751, + "pass@16": 0.9980793155373998, + "pass@32": 0.9985958445064392, + "pass@64": 0.9989373450806047, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99903125, + "pass@2": 0.999744217519685, + "pass@4": 0.9999508669853767, + "pass@8": 0.9999979402424715, + "pass@16": 0.999999997827342, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997328125, + "pass@2": 0.9989947096456695, + "pass@4": 0.9997306202662166, + "pass@8": 0.9999745071242221, + "pass@16": 0.999999771487176, + "pass@32": 0.9999999999945146, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920390625, + "pass@2": 0.9953330462598428, + "pass@4": 0.9967845025309336, + "pass@8": 0.9976012535726033, + "pass@16": 0.9982235495134753, + "pass@32": 0.998668917653394, + "pass@64": 0.9989403788828518, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9933046875, + "pass@2": 0.9949777312992126, + "pass@4": 0.99584106167979, + "pass@8": 0.9966582316227469, + "pass@16": 0.9975285452183399, + "pass@32": 0.9985580655950304, + "pass@64": 0.9995722834629693, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9376640625, + "2": 1.0, + "3": 0.9988671875, + "4": 0.9998046875, + "5": 0.9938828125, + "6": 0.99903125, + "7": 0.997328125, + "8": 0.9920390625, + "9": 0.9933046875 + }, + "per_op_avg_response_len": { + "10": 281.436203125, + "2": 115.651125, + "3": 144.9524921875, + "4": 159.92903125, + "5": 173.2407421875, + "6": 198.9064375, + "7": 224.0198203125, + "8": 243.6603515625, + "9": 259.680265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.436203125, + "2": 115.651125, + "3": 144.9524921875, + "4": 159.92903125, + "5": 173.2407421875, + "6": 198.9064375, + "7": 224.0198203125, + "8": 243.6603515625, + "9": 259.680265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375452, + "answer_accuracy": 0.9906176122931442, + "avg_response_len": 190.6621812732185, + "resp_tokens_sum": 72262492, + "pass_at_k": { + "pass@1": 0.9906176122931442, + "pass@2": 0.9931028402034852, + "pass@4": 0.9946938578916233, + "pass@8": 0.9959625637696556, + "pass@16": 0.9970240525816514, + "pass@32": 0.9980046850722043, + "pass@64": 0.9988549123125451, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9435149336283186, + "pass@2": 0.954635492880868, + "pass@4": 0.9636101311451112, + "pass@8": 0.9714900775767249, + "pass@16": 0.9780677121400383, + "pass@32": 0.9842094464543852, + "pass@64": 0.9901773717362713, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997371941896025, + "pass@2": 0.9999939801102844, + "pass@4": 0.9999999982800315, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9909474206349206, + "pass@2": 0.994433508311461, + "pass@4": 0.9964954672332624, + "pass@8": 0.9978625121315736, + "pass@16": 0.9989254195211111, + "pass@32": 0.9997055216891618, + "pass@64": 0.9999901718594374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992281062874252, + "pass@2": 0.9999447463341034, + "pass@4": 0.9999994695648073, + "pass@8": 0.9999999999730462, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997133875739645, + "pass@2": 0.9989840801612077, + "pass@4": 0.9997141855973625, + "pass@8": 0.9999752317035955, + "pass@16": 0.9999998695305009, + "pass@32": 0.999999999999429, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951923076923077, + "pass@2": 0.9986281199525542, + "pass@4": 0.9998137005710825, + "pass@8": 0.9999957663923499, + "pass@16": 0.9999999980003716, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.991637323943662, + "pass@2": 0.9929615587224134, + "pass@4": 0.9940380704172542, + "pass@8": 0.9954732852979431, + "pass@16": 0.9970755009921725, + "pass@32": 0.9986975516024429, + "pass@64": 0.9998376400088083, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386398, + "answer_accuracy": 0.9891003849934469, + "avg_response_len": 198.11501167267366, + "resp_tokens_sum": 77394818, + "pass_at_k": { + "pass@1": 0.9891003849934469, + "pass@2": 0.9915418265162386, + "pass@4": 0.9936347597769153, + "pass@8": 0.9953291833473309, + "pass@16": 0.9965900722029576, + "pass@32": 0.9976401391302876, + "pass@64": 0.9985809441882365, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9211320935582822, + "pass@2": 0.9365198994009952, + "pass@4": 0.9526557933134124, + "pass@8": 0.9657088477738169, + "pass@16": 0.9746686601446526, + "pass@32": 0.9816210687882091, + "pass@64": 0.9878415021526776, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9968513257575757, + "pass@2": 0.9974964954664758, + "pass@4": 0.9979660738430424, + "pass@8": 0.9986534604223928, + "pass@16": 0.9994358706064412, + "pass@32": 0.9999202169903283, + "pass@64": 0.9999995805383678, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962635869565217, + "pass@2": 0.9972030839895015, + "pass@4": 0.9974499779918814, + "pass@8": 0.9977680587306922, + "pass@16": 0.9983124881128989, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998209169054442, + "pass@2": 0.9999901292782528, + "pass@4": 0.9999999811986252, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987129696531792, + "pass@2": 0.9996746421419143, + "pass@4": 0.9999665655051787, + "pass@8": 0.9999997068622991, + "pass@16": 0.9999999999917892, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9898767605633803, + "pass@2": 0.9936876178329822, + "pass@4": 0.9957979374937288, + "pass@8": 0.9972295980502696, + "pass@16": 0.9982648879987673, + "pass@32": 0.9991210789903305, + "pass@64": 0.9998322279837888, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966785179640718, + "pass@2": 0.9970903419538876, + "pass@4": 0.9971908938029453, + "pass@8": 0.997369925503324, + "pass@16": 0.9977102880852468, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378876, + "answer_accuracy": 0.9909503682624707, + "avg_response_len": 211.67684445095412, + "resp_tokens_sum": 80931678, + "pass_at_k": { + "pass@1": 0.9909503682624707, + "pass@2": 0.9937347847892042, + "pass@4": 0.9953174973064762, + "pass@8": 0.9963995568925159, + "pass@16": 0.9973760492109548, + "pass@32": 0.998299954731427, + "pass@64": 0.9990212234898364, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9478311567164179, + "pass@2": 0.9607738864731464, + "pass@4": 0.9701930362249498, + "pass@8": 0.9777165144485844, + "pass@16": 0.9850490284905359, + "pass@32": 0.9921181247105286, + "pass@64": 0.9976074742078478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999772230320699, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9994745979532164, + "pass@2": 0.9999316491688539, + "pass@4": 0.9999986720409949, + "pass@8": 0.9999999997423706, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941865808823529, + "pass@2": 0.9964205071792498, + "pass@4": 0.9969975253093362, + "pass@8": 0.9970582244883547, + "pass@16": 0.9970588234979403, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9979544558359621, + "pass@2": 0.9992621991355971, + "pass@4": 0.9998455858435678, + "pass@8": 0.9999935023705825, + "pass@16": 0.9999999931461893, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960195806962026, + "pass@2": 0.9982615961078442, + "pass@4": 0.9994898540174567, + "pass@8": 0.9999461399811752, + "pass@16": 0.9999994164199608, + "pass@32": 0.9999999999832518, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913898273273273, + "pass@2": 0.9939999054172283, + "pass@4": 0.9949980785059523, + "pass@8": 0.9957539554064397, + "pass@16": 0.9965150597531441, + "pass@32": 0.9969397435790592, + "pass@64": 0.9969968106564769, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9915846061093248, + "pass@2": 0.9950103014152974, + "pass@4": 0.9964494795867558, + "pass@8": 0.9972464958966785, + "pass@16": 0.9978505020760328, + "pass@32": 0.9986542070682007, + "pass@64": 0.999607565131529, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.87520703125, + "avg_response_len": 381.9587625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.87520703125, + "pass@2": 0.9134097563976206, + "pass@4": 0.9389906385264302, + "pass@8": 0.9564744699367604, + "pass@16": 0.9682517650380048, + "pass@32": 0.9763231330001859, + "pass@64": 0.9823030713945525, + "pass@128": 0.9872 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.963765625, + "pass@2": 0.9766643700787401, + "pass@4": 0.9859928733595799, + "pass@8": 0.992054479165027, + "pass@16": 0.9956266043079476, + "pass@32": 0.9978735764864717, + "pass@64": 0.9994439280077732, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9433984375, + "pass@2": 0.9650922736220469, + "pass@4": 0.9807797401574805, + "pass@8": 0.9904969636512383, + "pass@16": 0.9955106251734532, + "pass@32": 0.9977054641496763, + "pass@64": 0.9986237310309992, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.921484375, + "pass@2": 0.9494464812992126, + "pass@4": 0.9694928863892013, + "pass@8": 0.9822071144268547, + "pass@16": 0.9895445856143713, + "pass@32": 0.9937076581126917, + "pass@64": 0.9964932015477215, + "pass@128": 0.999 + }, + "14": { + "pass@1": 0.929890625, + "pass@2": 0.9545305118110228, + "pass@4": 0.9717102435320588, + "pass@8": 0.9830305223063855, + "pass@16": 0.9897045103398158, + "pass@32": 0.993748432511619, + "pass@64": 0.996034478261216, + "pass@128": 0.997 + }, + "15": { + "pass@1": 0.943859375, + "pass@2": 0.9686481299212606, + "pass@4": 0.9817905391826017, + "pass@8": 0.9884232897126618, + "pass@16": 0.9919737217953848, + "pass@32": 0.9940940056649882, + "pass@64": 0.9956257924169127, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.880328125, + "pass@2": 0.9154064960629927, + "pass@4": 0.9409161780089994, + "pass@8": 0.9584434565398636, + "pass@16": 0.9695788008442296, + "pass@32": 0.9770087416058377, + "pass@64": 0.9829134464000747, + "pass@128": 0.989 + }, + "17": { + "pass@1": 0.8787265625, + "pass@2": 0.9150996555118119, + "pass@4": 0.9382658082114738, + "pass@8": 0.9548448203139048, + "pass@16": 0.9666152118144856, + "pass@32": 0.9751872078009532, + "pass@64": 0.9821301084988351, + "pass@128": 0.989 + }, + "18": { + "pass@1": 0.852015625, + "pass@2": 0.8975530265748031, + "pass@4": 0.9259083386764154, + "pass@8": 0.9456634829663602, + "pass@16": 0.9593639044923457, + "pass@32": 0.9689978125277122, + "pass@64": 0.9762361355053494, + "pass@128": 0.982 + }, + "19": { + "pass@1": 0.73671875, + "pass@2": 0.8070583169291341, + "pass@4": 0.8543637754968135, + "pass@8": 0.8884492110887334, + "pass@16": 0.9139215521312221, + "pass@32": 0.933041870170071, + "pass@64": 0.9482299524420053, + "pass@128": 0.961 + }, + "20": { + "pass@1": 0.7018828125, + "pass@2": 0.7845983021653551, + "pass@4": 0.8406860022497188, + "pass@8": 0.8811313591965394, + "pass@16": 0.9106781338667324, + "pass@32": 0.9318665609718125, + "pass@64": 0.9472999398346291, + "pass@128": 0.959 + } + }, + "per_op_accuracy": { + "11": 0.963765625, + "12": 0.9433984375, + "13": 0.921484375, + "14": 0.929890625, + "15": 0.943859375, + "16": 0.880328125, + "17": 0.8787265625, + "18": 0.852015625, + "19": 0.73671875, + "20": 0.7018828125 + }, + "per_op_avg_response_len": { + "11": 300.9099140625, + "12": 313.3621328125, + "13": 334.3359921875, + "14": 361.20790625, + "15": 380.9531953125, + "16": 392.9651875, + "17": 414.252109375, + "18": 430.980015625, + "19": 434.77484375, + "20": 455.846328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 300.9099140625, + "12": 313.3621328125, + "13": 334.3359921875, + "14": 361.20790625, + "15": 380.9531953125, + "16": 392.9651875, + "17": 414.252109375, + "18": 430.980015625, + "19": 434.77484375, + "20": 455.846328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 375178, + "answer_accuracy": 0.8710484769687964, + "avg_response_len": 364.06427841753344, + "resp_tokens_sum": 156809766, + "pass_at_k": { + "pass@1": 0.8710484769687964, + "pass@2": 0.9108458936364412, + "pass@4": 0.9370204153622743, + "pass@8": 0.9543075918523237, + "pass@16": 0.9659068589578557, + "pass@32": 0.9739126413934297, + "pass@64": 0.979709942605778, + "pass@128": 0.9845468053491828 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9542315864022662, + "pass@2": 0.969464140327006, + "pass@4": 0.9810940441934841, + "pass@8": 0.9893951399204369, + "pass@16": 0.9945360284144443, + "pass@32": 0.9977323646041456, + "pass@64": 0.9996420458747602, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9427304025423728, + "pass@2": 0.9646079396325459, + "pass@4": 0.979820588898845, + "pass@8": 0.989370272818801, + "pass@16": 0.9949116559332181, + "pass@32": 0.9979107888230833, + "pass@64": 0.9992908315443338, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.914576802507837, + "pass@2": 0.9446635956359689, + "pass@4": 0.9665589881202152, + "pass@8": 0.979981660826714, + "pass@16": 0.9874232215331469, + "pass@32": 0.9921527270385655, + "pass@64": 0.996205173271306, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9100467289719626, + "pass@2": 0.9379875266759877, + "pass@4": 0.958589766816531, + "pass@8": 0.9731059061694559, + "pass@16": 0.9830554969430491, + "pass@32": 0.9893467916867982, + "pass@64": 0.992726298871782, + "pass@128": 0.9937694704049844 + }, + "15": { + "pass@1": 0.9557216401734104, + "pass@2": 0.9786739480451507, + "pass@4": 0.988442469889963, + "pass@8": 0.9914905405416587, + "pass@16": 0.992362600034752, + "pass@32": 0.9933207386637914, + "pass@64": 0.9949478858495289, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8498739919354839, + "pass@2": 0.8913004826009657, + "pass@4": 0.9222687491684507, + "pass@8": 0.9436176461985871, + "pass@16": 0.9585810148724633, + "pass@32": 0.9695494764061836, + "pass@64": 0.9774985888021867, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8969698338150289, + "pass@2": 0.9344089225570076, + "pass@4": 0.956744095300948, + "pass@8": 0.97040257318339, + "pass@16": 0.9791461578401852, + "pass@32": 0.9845951312505338, + "pass@64": 0.9879973643102052, + "pass@128": 0.9913294797687862 + }, + "18": { + "pass@1": 0.8265625, + "pass@2": 0.8718394966254224, + "pass@4": 0.9008116189940538, + "pass@8": 0.9218754716646158, + "pass@16": 0.9370514134261215, + "pass@32": 0.9485340476411356, + "pass@64": 0.9583897263711767, + "pass@128": 0.9685714285714285 + }, + "19": { + "pass@1": 0.7230902777777778, + "pass@2": 0.7983750236456933, + "pass@4": 0.8509800430351607, + "pass@8": 0.8887121617215229, + "pass@16": 0.9156199204084648, + "pass@32": 0.9346616186743513, + "pass@64": 0.9488236564767019, + "pass@128": 0.960960960960961 + }, + "20": { + "pass@1": 0.7268909534534534, + "pass@2": 0.8097002021706746, + "pass@4": 0.8596189023669333, + "pass@8": 0.8914002622308476, + "pass@16": 0.9137724953284576, + "pass@32": 0.9295212910338141, + "pass@64": 0.9403257272551829, + "pass@128": 0.948948948948949 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 380143, + "answer_accuracy": 0.883100561254832, + "avg_response_len": 377.4973029103479, + "resp_tokens_sum": 162498999, + "pass_at_k": { + "pass@1": 0.883100561254832, + "pass@2": 0.9181039364810697, + "pass@4": 0.9419203114775695, + "pass@8": 0.958884741430148, + "pass@16": 0.970336246345682, + "pass@32": 0.9778981665204327, + "pass@64": 0.9835902449864664, + "pass@128": 0.9887005649717514 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.973621046686747, + "pass@2": 0.9814229852480792, + "pass@4": 0.9863388201286587, + "pass@8": 0.9898787999691451, + "pass@16": 0.993064840318199, + "pass@32": 0.9960242380201543, + "pass@64": 0.9987056869995018, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9421532131661442, + "pass@2": 0.9639698985510823, + "pass@4": 0.9794393557388399, + "pass@8": 0.9891352473786128, + "pass@16": 0.99374911404456, + "pass@32": 0.9954805067246403, + "pass@64": 0.9964824145461987, + "pass@128": 0.9968652037617555 + }, + "13": { + "pass@1": 0.9447435224089635, + "pass@2": 0.9655367068087075, + "pass@4": 0.9791388313855723, + "pass@8": 0.9881338330670343, + "pass@16": 0.9938460585401995, + "pass@32": 0.9969037090102838, + "pass@64": 0.9985099383695012, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9357638888888888, + "pass@2": 0.9592812514389654, + "pass@4": 0.976130010064531, + "pass@8": 0.9864184088906494, + "pass@16": 0.9913667647399901, + "pass@32": 0.9940353616501455, + "pass@64": 0.9955747091064813, + "pass@128": 0.9970760233918129 + }, + "15": { + "pass@1": 0.9528645833333333, + "pass@2": 0.9747278394178001, + "pass@4": 0.9865421262683073, + "pass@8": 0.993495451097799, + "pass@16": 0.9974149003571077, + "pass@32": 0.99942427013389, + "pass@64": 0.999978739438185, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8898459757834758, + "pass@2": 0.918867913946654, + "pass@4": 0.940958512130428, + "pass@8": 0.9576799933576514, + "pass@16": 0.9685457081346035, + "pass@32": 0.9757692082156921, + "pass@64": 0.9819115581804977, + "pass@128": 0.9886039886039886 + }, + "17": { + "pass@1": 0.8886354813664596, + "pass@2": 0.9250734368122461, + "pass@4": 0.9469407821499012, + "pass@8": 0.9609358541243732, + "pass@16": 0.9696919174554715, + "pass@32": 0.9768628206172025, + "pass@64": 0.9848878654881961, + "pass@128": 0.9937888198757764 + }, + "18": { + "pass@1": 0.865234375, + "pass@2": 0.9126758626679023, + "pass@4": 0.9434961421182645, + "pass@8": 0.9639815917003957, + "pass@16": 0.9758078148025395, + "pass@32": 0.9821697623332426, + "pass@64": 0.9862873674165625, + "pass@128": 0.9882352941176471 + }, + "19": { + "pass@1": 0.7456752232142857, + "pass@2": 0.8095109937429702, + "pass@4": 0.8518101247388715, + "pass@8": 0.8841118867969516, + "pass@16": 0.9092287569498064, + "pass@32": 0.9274672406486854, + "pass@64": 0.9428158675268603, + "pass@128": 0.9583333333333334 + }, + "20": { + "pass@1": 0.6919910179640718, + "pass@2": 0.7697962834174169, + "pass@4": 0.8282520268798735, + "pass@8": 0.8748525418555514, + "pass@16": 0.9103714017754003, + "pass@32": 0.9346016347947679, + "pass@64": 0.9506273862852723, + "pass@128": 0.9640718562874252 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 364944, + "answer_accuracy": 0.8713707212713936, + "avg_response_len": 404.94740172295235, + "resp_tokens_sum": 169598451, + "pass_at_k": { + "pass@1": 0.8713707212713936, + "pass@2": 0.9112217584058534, + "pass@4": 0.9380057090682301, + "pass@8": 0.9562256318321845, + "pass@16": 0.9685208659921463, + "pass@32": 0.9771833000320068, + "pass@64": 0.983646932514538, + "pass@128": 0.9883863080684596 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9640625, + "pass@2": 0.9797177696537934, + "pass@4": 0.9911180554811603, + "pass@8": 0.9973277243916092, + "pass@16": 0.9995487596571645, + "pass@32": 0.9999809674873567, + "pass@64": 0.99999999279412, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9453363914373089, + "pass@2": 0.9667115148450481, + "pass@4": 0.9831256795308844, + "pass@8": 0.9930450861150011, + "pass@16": 0.9978774666448946, + "pass@32": 0.9996537102787295, + "pass@64": 0.999990478361063, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9026572145061729, + "pass@2": 0.9364265122241666, + "pass@4": 0.9617531060932193, + "pass@8": 0.9778678586981546, + "pass@16": 0.9868936266865624, + "pass@32": 0.9917170187367842, + "pass@64": 0.9945546397415536, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9428319732937686, + "pass@2": 0.9654668479403722, + "pass@4": 0.9797224478987603, + "pass@8": 0.9890457880872057, + "pass@16": 0.994350926646944, + "pass@32": 0.9976499071092195, + "pass@64": 0.9996521834093685, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9220196759259259, + "pass@2": 0.9517492496597652, + "pass@4": 0.9698473547288068, + "pass@8": 0.9799816784659696, + "pass@16": 0.9860164971157882, + "pass@32": 0.9894908053800399, + "pass@64": 0.9919161725258471, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8983222713864307, + "pass@2": 0.9338663972313197, + "pass@4": 0.9579245663982268, + "pass@8": 0.9727914706482771, + "pass@16": 0.9807054356298547, + "pass@32": 0.9851133092513668, + "pass@64": 0.9889024393806544, + "pass@128": 0.9941002949852508 + }, + "17": { + "pass@1": 0.8501035391566265, + "pass@2": 0.8853027760648893, + "pass@4": 0.9105946367020382, + "pass@8": 0.9327234486879625, + "pass@16": 0.9505718186178311, + "pass@32": 0.9637574221371371, + "pass@64": 0.9733407462659791, + "pass@128": 0.9819277108433735 + }, + "18": { + "pass@1": 0.8662550403225806, + "pass@2": 0.9099980949961902, + "pass@4": 0.9349534958331817, + "pass@8": 0.9524300861471281, + "pass@16": 0.9665201701946451, + "pass@32": 0.9776554085806844, + "pass@64": 0.9853613753348571, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7413378021148036, + "pass@2": 0.8133043509289437, + "pass@4": 0.8603601790146319, + "pass@8": 0.8925875144159812, + "pass@16": 0.9169766051361591, + "pass@32": 0.9370711730923081, + "pass@64": 0.9531285297469417, + "pass@128": 0.9637462235649547 + }, + "20": { + "pass@1": 0.6867961711711712, + "pass@2": 0.7743428714147219, + "pass@4": 0.8342244167677225, + "pass@8": 0.8771601288105502, + "pass@16": 0.9078914256137317, + "pass@32": 0.9314685436519544, + "pass@64": 0.9509367136317464, + "pass@128": 0.963963963963964 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9296837993421052, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9296837993421052, + "pass@2": 0.9510077121321959, + "pass@4": 0.9653042920934156, + "pass@8": 0.9751463155107143, + "pass@16": 0.9818663699700375, + "pass@32": 0.9865812037846895, + "pass@64": 0.9901255550278746, + "pass@128": 0.9929473684210526 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9376640625, + "pass@2": 0.9507861712598423, + "pass@4": 0.9622442902137232, + "pass@8": 0.9716912530130499, + "pass@16": 0.9792983621669582, + "pass@32": 0.9860150425510199, + "pass@64": 0.9919049625799984, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988671875, + "pass@2": 0.999171875, + "pass@4": 0.9993288038057743, + "pass@8": 0.9995556419393896, + "pass@16": 0.9998138373001255, + "pass@32": 0.9999736716068084, + "pass@64": 0.9999998615776614, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998046875, + "pass@2": 0.999976624015748, + "pass@4": 0.9999995458380202, + "pass@8": 0.9999999999118908, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9938828125, + "pass@2": 0.9960645915354331, + "pass@4": 0.9969954731908509, + "pass@8": 0.9975564679095751, + "pass@16": 0.9980793155373998, + "pass@32": 0.9985958445064392, + "pass@64": 0.9989373450806047, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99903125, + "pass@2": 0.999744217519685, + "pass@4": 0.9999508669853767, + "pass@8": 0.9999979402424715, + "pass@16": 0.999999997827342, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997328125, + "pass@2": 0.9989947096456695, + "pass@4": 0.9997306202662166, + "pass@8": 0.9999745071242221, + "pass@16": 0.999999771487176, + "pass@32": 0.9999999999945146, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920390625, + "pass@2": 0.9953330462598428, + "pass@4": 0.9967845025309336, + "pass@8": 0.9976012535726033, + "pass@16": 0.9982235495134753, + "pass@32": 0.998668917653394, + "pass@64": 0.9989403788828518, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9933046875, + "pass@2": 0.9949777312992126, + "pass@4": 0.99584106167979, + "pass@8": 0.9966582316227469, + "pass@16": 0.9975285452183399, + "pass@32": 0.9985580655950304, + "pass@64": 0.9995722834629693, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.963765625, + "pass@2": 0.9766643700787401, + "pass@4": 0.9859928733595799, + "pass@8": 0.992054479165027, + "pass@16": 0.9956266043079476, + "pass@32": 0.9978735764864717, + "pass@64": 0.9994439280077732, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9433984375, + "pass@2": 0.9650922736220469, + "pass@4": 0.9807797401574805, + "pass@8": 0.9904969636512383, + "pass@16": 0.9955106251734532, + "pass@32": 0.9977054641496763, + "pass@64": 0.9986237310309992, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.921484375, + "pass@2": 0.9494464812992126, + "pass@4": 0.9694928863892013, + "pass@8": 0.9822071144268547, + "pass@16": 0.9895445856143713, + "pass@32": 0.9937076581126917, + "pass@64": 0.9964932015477215, + "pass@128": 0.999 + }, + "14": { + "pass@1": 0.929890625, + "pass@2": 0.9545305118110228, + "pass@4": 0.9717102435320588, + "pass@8": 0.9830305223063855, + "pass@16": 0.9897045103398158, + "pass@32": 0.993748432511619, + "pass@64": 0.996034478261216, + "pass@128": 0.997 + }, + "15": { + "pass@1": 0.943859375, + "pass@2": 0.9686481299212606, + "pass@4": 0.9817905391826017, + "pass@8": 0.9884232897126618, + "pass@16": 0.9919737217953848, + "pass@32": 0.9940940056649882, + "pass@64": 0.9956257924169127, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.880328125, + "pass@2": 0.9154064960629927, + "pass@4": 0.9409161780089994, + "pass@8": 0.9584434565398636, + "pass@16": 0.9695788008442296, + "pass@32": 0.9770087416058377, + "pass@64": 0.9829134464000747, + "pass@128": 0.989 + }, + "17": { + "pass@1": 0.8787265625, + "pass@2": 0.9150996555118119, + "pass@4": 0.9382658082114738, + "pass@8": 0.9548448203139048, + "pass@16": 0.9666152118144856, + "pass@32": 0.9751872078009532, + "pass@64": 0.9821301084988351, + "pass@128": 0.989 + }, + "18": { + "pass@1": 0.852015625, + "pass@2": 0.8975530265748031, + "pass@4": 0.9259083386764154, + "pass@8": 0.9456634829663602, + "pass@16": 0.9593639044923457, + "pass@32": 0.9689978125277122, + "pass@64": 0.9762361355053494, + "pass@128": 0.982 + }, + "19": { + "pass@1": 0.73671875, + "pass@2": 0.8070583169291341, + "pass@4": 0.8543637754968135, + "pass@8": 0.8884492110887334, + "pass@16": 0.9139215521312221, + "pass@32": 0.933041870170071, + "pass@64": 0.9482299524420053, + "pass@128": 0.961 + }, + "20": { + "pass@1": 0.7018828125, + "pass@2": 0.7845983021653551, + "pass@4": 0.8406860022497188, + "pass@8": 0.8811313591965394, + "pass@16": 0.9106781338667324, + "pass@32": 0.9318665609718125, + "pass@64": 0.9472999398346291, + "pass@128": 0.959 + } + }, + "per_op_accuracy": { + "10": 0.9376640625, + "2": 1.0, + "3": 0.9988671875, + "4": 0.9998046875, + "5": 0.9938828125, + "6": 0.99903125, + "7": 0.997328125, + "8": 0.9920390625, + "9": 0.9933046875, + "11": 0.963765625, + "12": 0.9433984375, + "13": 0.921484375, + "14": 0.929890625, + "15": 0.943859375, + "16": 0.880328125, + "17": 0.8787265625, + "18": 0.852015625, + "19": 0.73671875, + "20": 0.7018828125 + }, + "per_op_avg_response_len": { + "10": 281.436203125, + "2": 115.651125, + "3": 144.9524921875, + "4": 159.92903125, + "5": 173.2407421875, + "6": 198.9064375, + "7": 224.0198203125, + "8": 243.6603515625, + "9": 259.680265625, + "11": 300.9099140625, + "12": 313.3621328125, + "13": 334.3359921875, + "14": 361.20790625, + "15": 380.9531953125, + "16": 392.9651875, + "17": 414.252109375, + "18": 430.980015625, + "19": 434.77484375, + "20": 455.846328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.436203125, + "2": 115.651125, + "3": 144.9524921875, + "4": 159.92903125, + "5": 173.2407421875, + "6": 198.9064375, + "7": 224.0198203125, + "8": 243.6603515625, + "9": 259.680265625, + "11": 300.9099140625, + "12": 313.3621328125, + "13": 334.3359921875, + "14": 361.20790625, + "15": 380.9531953125, + "16": 392.9651875, + "17": 414.252109375, + "18": 430.980015625, + "19": 434.77484375, + "20": 455.846328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 750630, + "answer_accuracy": 0.9270149976288334, + "avg_response_len": 282.90025539440404, + "resp_tokens_sum": 229072258, + "pass_at_k": { + "pass@1": 0.9270149976288334, + "pass@2": 0.9493477619236584, + "pass@4": 0.96401552496224, + "pass@8": 0.9738049633109391, + "pass@16": 0.980471830554453, + "pass@32": 0.9851893630710853, + "pass@64": 0.9886710958308421, + "pass@128": 0.9914638001896934 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9435149336283186, + "pass@2": 0.954635492880868, + "pass@4": 0.9636101311451112, + "pass@8": 0.9714900775767249, + "pass@16": 0.9780677121400383, + "pass@32": 0.9842094464543852, + "pass@64": 0.9901773717362713, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997371941896025, + "pass@2": 0.9999939801102844, + "pass@4": 0.9999999982800315, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9909474206349206, + "pass@2": 0.994433508311461, + "pass@4": 0.9964954672332624, + "pass@8": 0.9978625121315736, + "pass@16": 0.9989254195211111, + "pass@32": 0.9997055216891618, + "pass@64": 0.9999901718594374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992281062874252, + "pass@2": 0.9999447463341034, + "pass@4": 0.9999994695648073, + "pass@8": 0.9999999999730462, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997133875739645, + "pass@2": 0.9989840801612077, + "pass@4": 0.9997141855973625, + "pass@8": 0.9999752317035955, + "pass@16": 0.9999998695305009, + "pass@32": 0.999999999999429, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951923076923077, + "pass@2": 0.9986281199525542, + "pass@4": 0.9998137005710825, + "pass@8": 0.9999957663923499, + "pass@16": 0.9999999980003716, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.991637323943662, + "pass@2": 0.9929615587224134, + "pass@4": 0.9940380704172542, + "pass@8": 0.9954732852979431, + "pass@16": 0.9970755009921725, + "pass@32": 0.9986975516024429, + "pass@64": 0.9998376400088083, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9542315864022662, + "pass@2": 0.969464140327006, + "pass@4": 0.9810940441934841, + "pass@8": 0.9893951399204369, + "pass@16": 0.9945360284144443, + "pass@32": 0.9977323646041456, + "pass@64": 0.9996420458747602, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9427304025423728, + "pass@2": 0.9646079396325459, + "pass@4": 0.979820588898845, + "pass@8": 0.989370272818801, + "pass@16": 0.9949116559332181, + "pass@32": 0.9979107888230833, + "pass@64": 0.9992908315443338, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.914576802507837, + "pass@2": 0.9446635956359689, + "pass@4": 0.9665589881202152, + "pass@8": 0.979981660826714, + "pass@16": 0.9874232215331469, + "pass@32": 0.9921527270385655, + "pass@64": 0.996205173271306, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9100467289719626, + "pass@2": 0.9379875266759877, + "pass@4": 0.958589766816531, + "pass@8": 0.9731059061694559, + "pass@16": 0.9830554969430491, + "pass@32": 0.9893467916867982, + "pass@64": 0.992726298871782, + "pass@128": 0.9937694704049844 + }, + "15": { + "pass@1": 0.9557216401734104, + "pass@2": 0.9786739480451507, + "pass@4": 0.988442469889963, + "pass@8": 0.9914905405416587, + "pass@16": 0.992362600034752, + "pass@32": 0.9933207386637914, + "pass@64": 0.9949478858495289, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8498739919354839, + "pass@2": 0.8913004826009657, + "pass@4": 0.9222687491684507, + "pass@8": 0.9436176461985871, + "pass@16": 0.9585810148724633, + "pass@32": 0.9695494764061836, + "pass@64": 0.9774985888021867, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8969698338150289, + "pass@2": 0.9344089225570076, + "pass@4": 0.956744095300948, + "pass@8": 0.97040257318339, + "pass@16": 0.9791461578401852, + "pass@32": 0.9845951312505338, + "pass@64": 0.9879973643102052, + "pass@128": 0.9913294797687862 + }, + "18": { + "pass@1": 0.8265625, + "pass@2": 0.8718394966254224, + "pass@4": 0.9008116189940538, + "pass@8": 0.9218754716646158, + "pass@16": 0.9370514134261215, + "pass@32": 0.9485340476411356, + "pass@64": 0.9583897263711767, + "pass@128": 0.9685714285714285 + }, + "19": { + "pass@1": 0.7230902777777778, + "pass@2": 0.7983750236456933, + "pass@4": 0.8509800430351607, + "pass@8": 0.8887121617215229, + "pass@16": 0.9156199204084648, + "pass@32": 0.9346616186743513, + "pass@64": 0.9488236564767019, + "pass@128": 0.960960960960961 + }, + "20": { + "pass@1": 0.7268909534534534, + "pass@2": 0.8097002021706746, + "pass@4": 0.8596189023669333, + "pass@8": 0.8914002622308476, + "pass@16": 0.9137724953284576, + "pass@32": 0.9295212910338141, + "pass@64": 0.9403257272551829, + "pass@128": 0.948948948948949 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 766541, + "answer_accuracy": 0.9335310307872174, + "avg_response_len": 292.15439521628997, + "resp_tokens_sum": 239893817, + "pass_at_k": { + "pass@1": 0.9335310307872174, + "pass@2": 0.953042742465052, + "pass@4": 0.9665239741758695, + "pass@8": 0.9762235468442136, + "pass@16": 0.9828267648985132, + "pass@32": 0.9872906061783097, + "pass@64": 0.9907222191039753, + "pass@128": 0.9937646141855028 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9211320935582822, + "pass@2": 0.9365198994009952, + "pass@4": 0.9526557933134124, + "pass@8": 0.9657088477738169, + "pass@16": 0.9746686601446526, + "pass@32": 0.9816210687882091, + "pass@64": 0.9878415021526776, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9968513257575757, + "pass@2": 0.9974964954664758, + "pass@4": 0.9979660738430424, + "pass@8": 0.9986534604223928, + "pass@16": 0.9994358706064412, + "pass@32": 0.9999202169903283, + "pass@64": 0.9999995805383678, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962635869565217, + "pass@2": 0.9972030839895015, + "pass@4": 0.9974499779918814, + "pass@8": 0.9977680587306922, + "pass@16": 0.9983124881128989, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998209169054442, + "pass@2": 0.9999901292782528, + "pass@4": 0.9999999811986252, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987129696531792, + "pass@2": 0.9996746421419143, + "pass@4": 0.9999665655051787, + "pass@8": 0.9999997068622991, + "pass@16": 0.9999999999917892, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9898767605633803, + "pass@2": 0.9936876178329822, + "pass@4": 0.9957979374937288, + "pass@8": 0.9972295980502696, + "pass@16": 0.9982648879987673, + "pass@32": 0.9991210789903305, + "pass@64": 0.9998322279837888, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966785179640718, + "pass@2": 0.9970903419538876, + "pass@4": 0.9971908938029453, + "pass@8": 0.997369925503324, + "pass@16": 0.9977102880852468, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.973621046686747, + "pass@2": 0.9814229852480792, + "pass@4": 0.9863388201286587, + "pass@8": 0.9898787999691451, + "pass@16": 0.993064840318199, + "pass@32": 0.9960242380201543, + "pass@64": 0.9987056869995018, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9421532131661442, + "pass@2": 0.9639698985510823, + "pass@4": 0.9794393557388399, + "pass@8": 0.9891352473786128, + "pass@16": 0.99374911404456, + "pass@32": 0.9954805067246403, + "pass@64": 0.9964824145461987, + "pass@128": 0.9968652037617555 + }, + "13": { + "pass@1": 0.9447435224089635, + "pass@2": 0.9655367068087075, + "pass@4": 0.9791388313855723, + "pass@8": 0.9881338330670343, + "pass@16": 0.9938460585401995, + "pass@32": 0.9969037090102838, + "pass@64": 0.9985099383695012, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9357638888888888, + "pass@2": 0.9592812514389654, + "pass@4": 0.976130010064531, + "pass@8": 0.9864184088906494, + "pass@16": 0.9913667647399901, + "pass@32": 0.9940353616501455, + "pass@64": 0.9955747091064813, + "pass@128": 0.9970760233918129 + }, + "15": { + "pass@1": 0.9528645833333333, + "pass@2": 0.9747278394178001, + "pass@4": 0.9865421262683073, + "pass@8": 0.993495451097799, + "pass@16": 0.9974149003571077, + "pass@32": 0.99942427013389, + "pass@64": 0.999978739438185, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8898459757834758, + "pass@2": 0.918867913946654, + "pass@4": 0.940958512130428, + "pass@8": 0.9576799933576514, + "pass@16": 0.9685457081346035, + "pass@32": 0.9757692082156921, + "pass@64": 0.9819115581804977, + "pass@128": 0.9886039886039886 + }, + "17": { + "pass@1": 0.8886354813664596, + "pass@2": 0.9250734368122461, + "pass@4": 0.9469407821499012, + "pass@8": 0.9609358541243732, + "pass@16": 0.9696919174554715, + "pass@32": 0.9768628206172025, + "pass@64": 0.9848878654881961, + "pass@128": 0.9937888198757764 + }, + "18": { + "pass@1": 0.865234375, + "pass@2": 0.9126758626679023, + "pass@4": 0.9434961421182645, + "pass@8": 0.9639815917003957, + "pass@16": 0.9758078148025395, + "pass@32": 0.9821697623332426, + "pass@64": 0.9862873674165625, + "pass@128": 0.9882352941176471 + }, + "19": { + "pass@1": 0.7456752232142857, + "pass@2": 0.8095109937429702, + "pass@4": 0.8518101247388715, + "pass@8": 0.8841118867969516, + "pass@16": 0.9092287569498064, + "pass@32": 0.9274672406486854, + "pass@64": 0.9428158675268603, + "pass@128": 0.9583333333333334 + }, + "20": { + "pass@1": 0.6919910179640718, + "pass@2": 0.7697962834174169, + "pass@4": 0.8282520268798735, + "pass@8": 0.8748525418555514, + "pass@16": 0.9103714017754003, + "pass@32": 0.9346016347947679, + "pass@64": 0.9506273862852723, + "pass@128": 0.9640718562874252 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 743820, + "answer_accuracy": 0.9284380492091389, + "avg_response_len": 312.7123554581403, + "resp_tokens_sum": 250530129, + "pass_at_k": { + "pass@1": 0.9284380492091389, + "pass@2": 0.9505996797682092, + "pass@4": 0.9653567733704562, + "pass@8": 0.9753979459646634, + "pass@16": 0.98229150543528, + "pass@32": 0.9872608599596581, + "pass@64": 0.9909840482108517, + "pass@128": 0.9936092027480428 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9478311567164179, + "pass@2": 0.9607738864731464, + "pass@4": 0.9701930362249498, + "pass@8": 0.9777165144485844, + "pass@16": 0.9850490284905359, + "pass@32": 0.9921181247105286, + "pass@64": 0.9976074742078478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999772230320699, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9994745979532164, + "pass@2": 0.9999316491688539, + "pass@4": 0.9999986720409949, + "pass@8": 0.9999999997423706, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941865808823529, + "pass@2": 0.9964205071792498, + "pass@4": 0.9969975253093362, + "pass@8": 0.9970582244883547, + "pass@16": 0.9970588234979403, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9979544558359621, + "pass@2": 0.9992621991355971, + "pass@4": 0.9998455858435678, + "pass@8": 0.9999935023705825, + "pass@16": 0.9999999931461893, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960195806962026, + "pass@2": 0.9982615961078442, + "pass@4": 0.9994898540174567, + "pass@8": 0.9999461399811752, + "pass@16": 0.9999994164199608, + "pass@32": 0.9999999999832518, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913898273273273, + "pass@2": 0.9939999054172283, + "pass@4": 0.9949980785059523, + "pass@8": 0.9957539554064397, + "pass@16": 0.9965150597531441, + "pass@32": 0.9969397435790592, + "pass@64": 0.9969968106564769, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9915846061093248, + "pass@2": 0.9950103014152974, + "pass@4": 0.9964494795867558, + "pass@8": 0.9972464958966785, + "pass@16": 0.9978505020760328, + "pass@32": 0.9986542070682007, + "pass@64": 0.999607565131529, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9640625, + "pass@2": 0.9797177696537934, + "pass@4": 0.9911180554811603, + "pass@8": 0.9973277243916092, + "pass@16": 0.9995487596571645, + "pass@32": 0.9999809674873567, + "pass@64": 0.99999999279412, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9453363914373089, + "pass@2": 0.9667115148450481, + "pass@4": 0.9831256795308844, + "pass@8": 0.9930450861150011, + "pass@16": 0.9978774666448946, + "pass@32": 0.9996537102787295, + "pass@64": 0.999990478361063, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9026572145061729, + "pass@2": 0.9364265122241666, + "pass@4": 0.9617531060932193, + "pass@8": 0.9778678586981546, + "pass@16": 0.9868936266865624, + "pass@32": 0.9917170187367842, + "pass@64": 0.9945546397415536, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9428319732937686, + "pass@2": 0.9654668479403722, + "pass@4": 0.9797224478987603, + "pass@8": 0.9890457880872057, + "pass@16": 0.994350926646944, + "pass@32": 0.9976499071092195, + "pass@64": 0.9996521834093685, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9220196759259259, + "pass@2": 0.9517492496597652, + "pass@4": 0.9698473547288068, + "pass@8": 0.9799816784659696, + "pass@16": 0.9860164971157882, + "pass@32": 0.9894908053800399, + "pass@64": 0.9919161725258471, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8983222713864307, + "pass@2": 0.9338663972313197, + "pass@4": 0.9579245663982268, + "pass@8": 0.9727914706482771, + "pass@16": 0.9807054356298547, + "pass@32": 0.9851133092513668, + "pass@64": 0.9889024393806544, + "pass@128": 0.9941002949852508 + }, + "17": { + "pass@1": 0.8501035391566265, + "pass@2": 0.8853027760648893, + "pass@4": 0.9105946367020382, + "pass@8": 0.9327234486879625, + "pass@16": 0.9505718186178311, + "pass@32": 0.9637574221371371, + "pass@64": 0.9733407462659791, + "pass@128": 0.9819277108433735 + }, + "18": { + "pass@1": 0.8662550403225806, + "pass@2": 0.9099980949961902, + "pass@4": 0.9349534958331817, + "pass@8": 0.9524300861471281, + "pass@16": 0.9665201701946451, + "pass@32": 0.9776554085806844, + "pass@64": 0.9853613753348571, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7413378021148036, + "pass@2": 0.8133043509289437, + "pass@4": 0.8603601790146319, + "pass@8": 0.8925875144159812, + "pass@16": 0.9169766051361591, + "pass@32": 0.9370711730923081, + "pass@64": 0.9531285297469417, + "pass@128": 0.9637462235649547 + }, + "20": { + "pass@1": 0.6867961711711712, + "pass@2": 0.7743428714147219, + "pass@4": 0.8342244167677225, + "pass@8": 0.8771601288105502, + "pass@16": 0.9078914256137317, + "pass@32": 0.9314685436519544, + "pass@64": 0.9509367136317464, + "pass@128": 0.963963963963964 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3987e58ad9456ec4e3ef96a777bafc5635517315 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4658f24b4ebe375eb7ceee63f9db7393d364d3658e9fa43e64e0b6305d883680 +size 2723279001 diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2bd29188232c61cfae73b8834b81ebb2e1a77309 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive", "results_dir": "results/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1019512, "value_mismatch": 12612, "dependency_mismatch": 129967, "answer_mismatch": 11274, "extra_nodes": 15910, "missing_nodes": 12824, "perfect_with_extra": 1419, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 334036, "dependency_mismatch": 43833, "extra_nodes": 7352, "missing_nodes": 6550, "value_mismatch": 4119, "answer_mismatch": 3556, "perfect_with_extra": 357}, "teachers_in_school": {"total": 390656, "perfect": 346374, "value_mismatch": 4437, "dependency_mismatch": 43629, "answer_mismatch": 4258, "extra_nodes": 4389, "missing_nodes": 3728, "perfect_with_extra": 465}, "movie_festival_awards": {"total": 382336, "perfect": 339102, "dependency_mismatch": 42505, "value_mismatch": 4056, "answer_mismatch": 3460, "extra_nodes": 4169, "perfect_with_extra": 597, "missing_nodes": 2546}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 646443, "value_mismatch": 211614, "dependency_mismatch": 595028, "answer_mismatch": 159735, "extra_nodes": 62432, "missing_nodes": 128108, "perfect_with_extra": 469, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 212224, "extra_nodes": 32237, "perfect_with_extra": 79, "dependency_mismatch": 204730, "missing_nodes": 53647, "value_mismatch": 73200, "answer_mismatch": 55542}, "movie_festival_awards": {"total": 418816, "perfect": 213552, "value_mismatch": 71108, "dependency_mismatch": 192019, "answer_mismatch": 53872, "extra_nodes": 14537, "perfect_with_extra": 215, "missing_nodes": 37651}, "teachers_in_school": {"total": 430464, "perfect": 220667, "dependency_mismatch": 198279, "extra_nodes": 15658, "missing_nodes": 36810, "value_mismatch": 67306, "answer_mismatch": 50321, "perfect_with_extra": 175}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1665955, "value_mismatch": 224226, "dependency_mismatch": 724995, "answer_mismatch": 171009, "extra_nodes": 78342, "missing_nodes": 140932, "perfect_with_extra": 1888, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 546260, "dependency_mismatch": 248563, "extra_nodes": 39589, "missing_nodes": 60197, "value_mismatch": 77319, "answer_mismatch": 59098, "perfect_with_extra": 436}, "teachers_in_school": {"total": 821120, "perfect": 567041, "value_mismatch": 71743, "dependency_mismatch": 241908, "answer_mismatch": 54579, "extra_nodes": 20047, "missing_nodes": 40538, "perfect_with_extra": 640}, "movie_festival_awards": {"total": 801152, "perfect": 552654, "dependency_mismatch": 234524, "value_mismatch": 75164, "answer_mismatch": 57332, "extra_nodes": 18706, "perfect_with_extra": 812, "missing_nodes": 40197}}}} diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1db64c0fc2fcd5b6151d878e7143b40d5a025683 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f190a24e3bc1368f9c0f978b11cc7d0709d4be6faa662837115995fad792ddc4 +size 373324592 diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3ab3a45de5604c744a7944443aeb256ef629ee7c --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c77d5c2dbbb0b0f544d6d3d4d246bc02a0d992838b0a77c750fa583209762b5 +size 422007946 diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/results.json b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..697a49ad0d706baaa3e5df15aec0182bd10b7175 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-39008_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.99375, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9492063492063492, + "pass@2": 0.9777777777777777, + "pass@4": 0.9904761904761905, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9670658682634731, + "pass@2": 0.9790419161676647, + "pass@4": 0.9820359281437125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8816568047337278, + "pass@2": 0.9289940828402367, + "pass@4": 0.9585798816568047, + "pass@8": 0.9881656804733728, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8301282051282052, + "pass@2": 0.9102564102564102, + "pass@4": 0.9423076923076923, + "pass@8": 0.9711538461538461, + "pass@16": 0.9935897435897436, + "pass@32": 0.9935897435897436, + "pass@64": 0.9967948717948718, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7492957746478873, + "pass@2": 0.8394366197183099, + "pass@4": 0.8929577464788733, + "pass@8": 0.9492957746478873, + "pass@16": 0.9859154929577465, + "pass@32": 0.9915492957746479, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.6194690265486725, + "pass@2": 0.6991150442477876, + "pass@4": 0.7876106194690266, + "pass@8": 0.8466076696165191, + "pass@16": 0.8908554572271387, + "pass@32": 0.9233038348082596, + "pass@64": 0.9380530973451328, + "pass@128": 0.9528023598820059 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 0.9969418960244648, + "4": 0.990625, + "5": 0.9492063492063492, + "6": 0.9670658682634731, + "7": 0.8816568047337278, + "8": 0.8301282051282052, + "9": 0.7492957746478873, + "10": 0.6194690265486725 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9777777777777777, + "6": 0.9790419161676647, + "7": 0.9289940828402367, + "8": 0.9102564102564102, + "9": 0.8394366197183099, + "10": 0.6991150442477876 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9904761904761905, + "6": 0.9820359281437125, + "7": 0.9585798816568047, + "8": 0.9423076923076923, + "9": 0.8929577464788733, + "10": 0.7876106194690266 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9881656804733728, + "8": 0.9711538461538461, + "9": 0.9492957746478873, + "10": 0.8466076696165191 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9859154929577465, + "10": 0.8908554572271387 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9935897435897436, + "9": 0.9915492957746479, + "10": 0.9233038348082596 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9380530973451328 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9528023598820059 + } + }, + "total_pass_at_k": { + "pass@1": 0.8841607565011821, + "pass@2": 0.9233367105707532, + "pass@4": 0.9486659912191827, + "pass@8": 0.971968929415738, + "pass@16": 0.9848024316109423, + "pass@32": 0.9895305639986491, + "pass@64": 0.9922323539344816, + "pass@128": 0.994258696386356 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9970760233918129, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9470588235294117, + "pass@2": 0.961764705882353, + "pass@4": 0.9882352941176471, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9716088328075709, + "pass@2": 0.9747634069400631, + "pass@4": 0.9905362776025236, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8386075949367089, + "pass@2": 0.8955696202531646, + "pass@4": 0.9525316455696202, + "pass@8": 0.9841772151898734, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8048048048048048, + "pass@2": 0.8828828828828829, + "pass@4": 0.9279279279279279, + "pass@8": 0.960960960960961, + "pass@16": 0.987987987987988, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7845659163987139, + "pass@2": 0.8585209003215434, + "pass@4": 0.9196141479099679, + "pass@8": 0.9485530546623794, + "pass@16": 0.9807073954983923, + "pass@32": 0.9839228295819936, + "pass@64": 0.9935691318327974, + "pass@128": 0.9935691318327974 + }, + "10": { + "pass@1": 0.5880597014925373, + "pass@2": 0.6686567164179105, + "pass@4": 0.7343283582089553, + "pass@8": 0.8388059701492537, + "pass@16": 0.9134328358208955, + "pass@32": 0.9402985074626866, + "pass@64": 0.9641791044776119, + "pass@128": 0.9731343283582089 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9883040935672515, + "5": 0.9470588235294117, + "6": 0.9716088328075709, + "7": 0.8386075949367089, + "8": 0.8048048048048048, + "9": 0.7845659163987139, + "10": 0.5880597014925373 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.961764705882353, + "6": 0.9747634069400631, + "7": 0.8955696202531646, + "8": 0.8828828828828829, + "9": 0.8585209003215434, + "10": 0.6686567164179105 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9905362776025236, + "7": 0.9525316455696202, + "8": 0.9279279279279279, + "9": 0.9196141479099679, + "10": 0.7343283582089553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.9841772151898734, + "8": 0.960960960960961, + "9": 0.9485530546623794, + "10": 0.8388059701492537 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 0.987987987987988, + "9": 0.9807073954983923, + "10": 0.9134328358208955 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9839228295819936, + "10": 0.9402985074626866 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9935691318327974, + "10": 0.9641791044776119 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9935691318327974, + "10": 0.9731343283582089 + } + }, + "total_pass_at_k": { + "pass@1": 0.8818212253096752, + "pass@2": 0.9166387679946435, + "pass@4": 0.9464345497154335, + "pass@8": 0.9698694342149313, + "pass@16": 0.9866086374288584, + "pass@32": 0.9909608302644795, + "pass@64": 0.9946434549715434, + "pass@128": 0.995647807164379 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 0.996969696969697, + "pass@8": 0.996969696969697, + "pass@16": 0.996969696969697, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 0.9881656804733728, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9391304347826087, + "pass@2": 0.9681159420289855, + "pass@4": 0.9884057971014493, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9656160458452722, + "pass@2": 0.9713467048710601, + "pass@4": 0.9828080229226361, + "pass@8": 0.9914040114613181, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757225433526011, + "pass@2": 0.9393063583815029, + "pass@4": 0.9682080924855492, + "pass@8": 0.9884393063583815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7971830985915493, + "pass@2": 0.8647887323943662, + "pass@4": 0.9380281690140845, + "pass@8": 0.9661971830985916, + "pass@16": 0.9943661971830986, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7365269461077845, + "pass@2": 0.7934131736526946, + "pass@4": 0.8772455089820359, + "pass@8": 0.9341317365269461, + "pass@16": 0.9730538922155688, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6625766871165644, + "pass@2": 0.7116564417177914, + "pass@4": 0.7852760736196319, + "pass@8": 0.8282208588957055, + "pass@16": 0.8773006134969326, + "pass@32": 0.9233128834355828, + "pass@64": 0.9478527607361963, + "pass@128": 0.950920245398773 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9822485207100592, + "5": 0.9391304347826087, + "6": 0.9656160458452722, + "7": 0.8757225433526011, + "8": 0.7971830985915493, + "9": 0.7365269461077845, + "10": 0.6625766871165644 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9881656804733728, + "5": 0.9681159420289855, + "6": 0.9713467048710601, + "7": 0.9393063583815029, + "8": 0.8647887323943662, + "9": 0.7934131736526946, + "10": 0.7116564417177914 + }, + "pass@4": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.9828080229226361, + "7": 0.9682080924855492, + "8": 0.9380281690140845, + "9": 0.8772455089820359, + "10": 0.7852760736196319 + }, + "pass@8": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.9884393063583815, + "8": 0.9661971830985916, + "9": 0.9341317365269461, + "10": 0.8282208588957055 + }, + "pass@16": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9943661971830986, + "9": 0.9730538922155688, + "10": 0.8773006134969326 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9233128834355828 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9478527607361963 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.950920245398773 + } + }, + "total_pass_at_k": { + "pass@1": 0.8843381389252949, + "pass@2": 0.9154652686762779, + "pass@4": 0.9492136304062909, + "pass@8": 0.9675622542595019, + "pass@16": 0.9823066841415465, + "pass@32": 0.991480996068152, + "pass@64": 0.994429882044561, + "pass@128": 0.9947575360419397 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.987, + "pass@2": 0.993, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.945, + "pass@2": 0.969, + "pass@4": 0.989, + "pass@8": 0.998, + "pass@16": 0.998, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.968, + "pass@2": 0.975, + "pass@4": 0.985, + "pass@8": 0.996, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.866, + "pass@2": 0.922, + "pass@4": 0.96, + "pass@8": 0.987, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.81, + "pass@2": 0.885, + "pass@4": 0.936, + "pass@8": 0.966, + "pass@16": 0.992, + "pass@32": 0.997, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.756, + "pass@2": 0.83, + "pass@4": 0.896, + "pass@8": 0.944, + "pass@16": 0.98, + "pass@32": 0.991, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "10": { + "pass@1": 0.623, + "pass@2": 0.693, + "pass@4": 0.769, + "pass@8": 0.838, + "pass@16": 0.894, + "pass@32": 0.929, + "pass@64": 0.95, + "pass@128": 0.959 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.997, + "4": 0.987, + "5": 0.945, + "6": 0.968, + "7": 0.866, + "8": 0.81, + "9": 0.756, + "10": 0.623 + }, + "pass@2": { + "2": 1.0, + "3": 0.999, + "4": 0.993, + "5": 0.969, + "6": 0.975, + "7": 0.922, + "8": 0.885, + "9": 0.83, + "10": 0.693 + }, + "pass@4": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.989, + "6": 0.985, + "7": 0.96, + "8": 0.936, + "9": 0.896, + "10": 0.769 + }, + "pass@8": { + "2": 1.0, + "3": 0.999, + "4": 1.0, + "5": 0.998, + "6": 0.996, + "7": 0.987, + "8": 0.966, + "9": 0.944, + "10": 0.838 + }, + "pass@16": { + "2": 1.0, + "3": 0.999, + "4": 1.0, + "5": 0.998, + "6": 0.999, + "7": 0.999, + "8": 0.992, + "9": 0.98, + "10": 0.894 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.997, + "9": 0.991, + "10": 0.929 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.997, + "10": 0.95 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.997, + "10": 0.959 + } + }, + "total_pass_at_k": { + "pass@1": 0.8834444444444445, + "pass@2": 0.9184444444444444, + "pass@4": 0.9481111111111111, + "pass@8": 0.9697777777777777, + "pass@16": 0.9845555555555555, + "pass@32": 0.9906666666666667, + "pass@64": 0.9937777777777778, + "pass@128": 0.9948888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-39008_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.56657223796034, + "pass@2": 0.6487252124645893, + "pass@4": 0.7648725212464589, + "pass@8": 0.8441926345609065, + "pass@16": 0.9065155807365439, + "pass@32": 0.9376770538243626, + "pass@64": 0.9660056657223796, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.6271186440677966, + "pass@2": 0.6977401129943502, + "pass@4": 0.751412429378531, + "pass@8": 0.8418079096045198, + "pass@16": 0.9067796610169492, + "pass@32": 0.9350282485875706, + "pass@64": 0.963276836158192, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.5768025078369906, + "pass@2": 0.6238244514106583, + "pass@4": 0.7021943573667712, + "pass@8": 0.8056426332288401, + "pass@16": 0.8620689655172413, + "pass@32": 0.890282131661442, + "pass@64": 0.9216300940438872, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.5264797507788161, + "pass@2": 0.5763239875389408, + "pass@4": 0.6199376947040498, + "pass@8": 0.6915887850467289, + "pass@16": 0.7538940809968847, + "pass@32": 0.8286604361370716, + "pass@64": 0.8785046728971962, + "pass@128": 0.9065420560747663 + }, + "15": { + "pass@1": 0.5028901734104047, + "pass@2": 0.5549132947976878, + "pass@4": 0.5924855491329479, + "pass@8": 0.653179190751445, + "pass@16": 0.7196531791907514, + "pass@32": 0.815028901734104, + "pass@64": 0.8901734104046243, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.4645161290322581, + "pass@2": 0.5193548387096775, + "pass@4": 0.5451612903225806, + "pass@8": 0.5903225806451613, + "pass@16": 0.6451612903225806, + "pass@32": 0.7225806451612903, + "pass@64": 0.7741935483870968, + "pass@128": 0.7935483870967742 + }, + "17": { + "pass@1": 0.5, + "pass@2": 0.5404624277456648, + "pass@4": 0.5838150289017341, + "pass@8": 0.6213872832369942, + "pass@16": 0.6936416184971098, + "pass@32": 0.7485549132947977, + "pass@64": 0.8092485549132948, + "pass@128": 0.8554913294797688 + }, + "18": { + "pass@1": 0.44, + "pass@2": 0.4942857142857143, + "pass@4": 0.5114285714285715, + "pass@8": 0.56, + "pass@16": 0.6171428571428571, + "pass@32": 0.6657142857142857, + "pass@64": 0.7371428571428571, + "pass@128": 0.7885714285714286 + }, + "19": { + "pass@1": 0.37237237237237236, + "pass@2": 0.4264264264264264, + "pass@4": 0.46846846846846846, + "pass@8": 0.5105105105105106, + "pass@16": 0.5375375375375375, + "pass@32": 0.5705705705705706, + "pass@64": 0.6186186186186187, + "pass@128": 0.6846846846846847 + }, + "20": { + "pass@1": 0.36936936936936937, + "pass@2": 0.42042042042042044, + "pass@4": 0.4444444444444444, + "pass@8": 0.4774774774774775, + "pass@16": 0.5255255255255256, + "pass@32": 0.5825825825825826, + "pass@64": 0.6516516516516516, + "pass@128": 0.6996996996996997 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.56657223796034, + "12": 0.6271186440677966, + "13": 0.5768025078369906, + "14": 0.5264797507788161, + "15": 0.5028901734104047, + "16": 0.4645161290322581, + "17": 0.5, + "18": 0.44, + "19": 0.37237237237237236, + "20": 0.36936936936936937 + }, + "pass@2": { + "11": 0.6487252124645893, + "12": 0.6977401129943502, + "13": 0.6238244514106583, + "14": 0.5763239875389408, + "15": 0.5549132947976878, + "16": 0.5193548387096775, + "17": 0.5404624277456648, + "18": 0.4942857142857143, + "19": 0.4264264264264264, + "20": 0.42042042042042044 + }, + "pass@4": { + "11": 0.7648725212464589, + "12": 0.751412429378531, + "13": 0.7021943573667712, + "14": 0.6199376947040498, + "15": 0.5924855491329479, + "16": 0.5451612903225806, + "17": 0.5838150289017341, + "18": 0.5114285714285715, + "19": 0.46846846846846846, + "20": 0.4444444444444444 + }, + "pass@8": { + "11": 0.8441926345609065, + "12": 0.8418079096045198, + "13": 0.8056426332288401, + "14": 0.6915887850467289, + "15": 0.653179190751445, + "16": 0.5903225806451613, + "17": 0.6213872832369942, + "18": 0.56, + "19": 0.5105105105105106, + "20": 0.4774774774774775 + }, + "pass@16": { + "11": 0.9065155807365439, + "12": 0.9067796610169492, + "13": 0.8620689655172413, + "14": 0.7538940809968847, + "15": 0.7196531791907514, + "16": 0.6451612903225806, + "17": 0.6936416184971098, + "18": 0.6171428571428571, + "19": 0.5375375375375375, + "20": 0.5255255255255256 + }, + "pass@32": { + "11": 0.9376770538243626, + "12": 0.9350282485875706, + "13": 0.890282131661442, + "14": 0.8286604361370716, + "15": 0.815028901734104, + "16": 0.7225806451612903, + "17": 0.7485549132947977, + "18": 0.6657142857142857, + "19": 0.5705705705705706, + "20": 0.5825825825825826 + }, + "pass@64": { + "11": 0.9660056657223796, + "12": 0.963276836158192, + "13": 0.9216300940438872, + "14": 0.8785046728971962, + "15": 0.8901734104046243, + "16": 0.7741935483870968, + "17": 0.8092485549132948, + "18": 0.7371428571428571, + "19": 0.6186186186186187, + "20": 0.6516516516516516 + }, + "pass@128": { + "11": 0.9801699716713881, + "12": 0.9774011299435028, + "13": 0.9498432601880877, + "14": 0.9065420560747663, + "15": 0.9508670520231214, + "16": 0.7935483870967742, + "17": 0.8554913294797688, + "18": 0.7885714285714286, + "19": 0.6846846846846847, + "20": 0.6996996996996997 + } + }, + "total_pass_at_k": { + "pass@1": 0.49539375928677565, + "pass@2": 0.5512630014858841, + "pass@4": 0.599702823179792, + "pass@8": 0.6609212481426449, + "pass@16": 0.7182763744427935, + "pass@32": 0.7708766716196137, + "pass@64": 0.8222882615156017, + "pass@128": 0.8600297176820209 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6126984126984127, + "pass@2": 0.6888888888888889, + "pass@4": 0.7746031746031746, + "pass@8": 0.8476190476190476, + "pass@16": 0.8984126984126984, + "pass@32": 0.946031746031746, + "pass@64": 0.9809523809523809, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.6330275229357798, + "pass@2": 0.7155963302752294, + "pass@4": 0.7737003058103975, + "pass@8": 0.8654434250764526, + "pass@16": 0.9051987767584098, + "pass@32": 0.9541284403669725, + "pass@64": 0.9755351681957186, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.5864197530864198, + "pass@2": 0.6203703703703703, + "pass@4": 0.7067901234567902, + "pass@8": 0.8055555555555556, + "pass@16": 0.8549382716049383, + "pass@32": 0.9012345679012346, + "pass@64": 0.9382716049382716, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.5459940652818991, + "pass@2": 0.5786350148367952, + "pass@4": 0.6112759643916914, + "pass@8": 0.6646884272997032, + "pass@16": 0.7418397626112759, + "pass@32": 0.8456973293768546, + "pass@64": 0.9020771513353115, + "pass@128": 0.9347181008902077 + }, + "15": { + "pass@1": 0.4783950617283951, + "pass@2": 0.5277777777777778, + "pass@4": 0.5679012345679012, + "pass@8": 0.6172839506172839, + "pass@16": 0.691358024691358, + "pass@32": 0.7870370370370371, + "pass@64": 0.8888888888888888, + "pass@128": 0.941358024691358 + }, + "16": { + "pass@1": 0.5103244837758112, + "pass@2": 0.5663716814159292, + "pass@4": 0.5958702064896755, + "pass@8": 0.6637168141592921, + "pass@16": 0.7109144542772862, + "pass@32": 0.7551622418879056, + "pass@64": 0.8082595870206489, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.5030120481927711, + "pass@2": 0.5421686746987951, + "pass@4": 0.5933734939759037, + "pass@8": 0.6295180722891566, + "pass@16": 0.6656626506024096, + "pass@32": 0.7018072289156626, + "pass@64": 0.7560240963855421, + "pass@128": 0.8102409638554217 + }, + "18": { + "pass@1": 0.4806451612903226, + "pass@2": 0.5193548387096775, + "pass@4": 0.5387096774193548, + "pass@8": 0.6064516129032258, + "pass@16": 0.6548387096774193, + "pass@32": 0.6838709677419355, + "pass@64": 0.7677419354838709, + "pass@128": 0.8354838709677419 + }, + "19": { + "pass@1": 0.38368580060422963, + "pass@2": 0.43202416918429004, + "pass@4": 0.48036253776435045, + "pass@8": 0.513595166163142, + "pass@16": 0.5558912386706949, + "pass@32": 0.6042296072507553, + "pass@64": 0.6646525679758308, + "pass@128": 0.7009063444108762 + }, + "20": { + "pass@1": 0.34234234234234234, + "pass@2": 0.3933933933933934, + "pass@4": 0.4264264264264264, + "pass@8": 0.45045045045045046, + "pass@16": 0.4924924924924925, + "pass@32": 0.5465465465465466, + "pass@64": 0.6036036036036037, + "pass@128": 0.6636636636636637 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6126984126984127, + "12": 0.6330275229357798, + "13": 0.5864197530864198, + "14": 0.5459940652818991, + "15": 0.4783950617283951, + "16": 0.5103244837758112, + "17": 0.5030120481927711, + "18": 0.4806451612903226, + "19": 0.38368580060422963, + "20": 0.34234234234234234 + }, + "pass@2": { + "11": 0.6888888888888889, + "12": 0.7155963302752294, + "13": 0.6203703703703703, + "14": 0.5786350148367952, + "15": 0.5277777777777778, + "16": 0.5663716814159292, + "17": 0.5421686746987951, + "18": 0.5193548387096775, + "19": 0.43202416918429004, + "20": 0.3933933933933934 + }, + "pass@4": { + "11": 0.7746031746031746, + "12": 0.7737003058103975, + "13": 0.7067901234567902, + "14": 0.6112759643916914, + "15": 0.5679012345679012, + "16": 0.5958702064896755, + "17": 0.5933734939759037, + "18": 0.5387096774193548, + "19": 0.48036253776435045, + "20": 0.4264264264264264 + }, + "pass@8": { + "11": 0.8476190476190476, + "12": 0.8654434250764526, + "13": 0.8055555555555556, + "14": 0.6646884272997032, + "15": 0.6172839506172839, + "16": 0.6637168141592921, + "17": 0.6295180722891566, + "18": 0.6064516129032258, + "19": 0.513595166163142, + "20": 0.45045045045045046 + }, + "pass@16": { + "11": 0.8984126984126984, + "12": 0.9051987767584098, + "13": 0.8549382716049383, + "14": 0.7418397626112759, + "15": 0.691358024691358, + "16": 0.7109144542772862, + "17": 0.6656626506024096, + "18": 0.6548387096774193, + "19": 0.5558912386706949, + "20": 0.4924924924924925 + }, + "pass@32": { + "11": 0.946031746031746, + "12": 0.9541284403669725, + "13": 0.9012345679012346, + "14": 0.8456973293768546, + "15": 0.7870370370370371, + "16": 0.7551622418879056, + "17": 0.7018072289156626, + "18": 0.6838709677419355, + "19": 0.6042296072507553, + "20": 0.5465465465465466 + }, + "pass@64": { + "11": 0.9809523809523809, + "12": 0.9755351681957186, + "13": 0.9382716049382716, + "14": 0.9020771513353115, + "15": 0.8888888888888888, + "16": 0.8082595870206489, + "17": 0.7560240963855421, + "18": 0.7677419354838709, + "19": 0.6646525679758308, + "20": 0.6036036036036037 + }, + "pass@128": { + "11": 0.9904761904761905, + "12": 0.981651376146789, + "13": 0.9537037037037037, + "14": 0.9347181008902077, + "15": 0.941358024691358, + "16": 0.8436578171091446, + "17": 0.8102409638554217, + "18": 0.8354838709677419, + "19": 0.7009063444108762, + "20": 0.6636636636636637 + } + }, + "total_pass_at_k": { + "pass@1": 0.5070293398533008, + "pass@2": 0.5577628361858191, + "pass@4": 0.6060513447432763, + "pass@8": 0.6653422982885085, + "pass@16": 0.7160757946210269, + "pass@32": 0.7716992665036675, + "pass@64": 0.8276283618581907, + "pass@128": 0.8646088019559902 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6596385542168675, + "pass@2": 0.713855421686747, + "pass@4": 0.7981927710843374, + "pass@8": 0.8554216867469879, + "pass@16": 0.9126506024096386, + "pass@32": 0.9457831325301205, + "pass@64": 0.9668674698795181, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.5956112852664577, + "pass@2": 0.6520376175548589, + "pass@4": 0.7304075235109718, + "pass@8": 0.8432601880877743, + "pass@16": 0.9122257053291536, + "pass@32": 0.9373040752351097, + "pass@64": 0.9529780564263323, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.6078431372549019, + "pass@2": 0.6470588235294118, + "pass@4": 0.7563025210084033, + "pass@8": 0.8291316526610645, + "pass@16": 0.8935574229691877, + "pass@32": 0.9243697478991597, + "pass@64": 0.9523809523809523, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.5584795321637427, + "pass@2": 0.5964912280701754, + "pass@4": 0.652046783625731, + "pass@8": 0.7309941520467836, + "pass@16": 0.8187134502923976, + "pass@32": 0.8888888888888888, + "pass@64": 0.935672514619883, + "pass@128": 0.956140350877193 + }, + "15": { + "pass@1": 0.5393939393939394, + "pass@2": 0.5696969696969697, + "pass@4": 0.6212121212121212, + "pass@8": 0.6787878787878788, + "pass@16": 0.7606060606060606, + "pass@32": 0.8333333333333334, + "pass@64": 0.8878787878787879, + "pass@128": 0.9333333333333333 + }, + "16": { + "pass@1": 0.5213675213675214, + "pass@2": 0.5555555555555556, + "pass@4": 0.5811965811965812, + "pass@8": 0.6381766381766382, + "pass@16": 0.6837606837606838, + "pass@32": 0.7435897435897436, + "pass@64": 0.8176638176638177, + "pass@128": 0.8547008547008547 + }, + "17": { + "pass@1": 0.42857142857142855, + "pass@2": 0.468944099378882, + "pass@4": 0.5186335403726708, + "pass@8": 0.5714285714285714, + "pass@16": 0.6366459627329193, + "pass@32": 0.7204968944099379, + "pass@64": 0.7950310559006211, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.4294117647058823, + "pass@2": 0.4676470588235294, + "pass@4": 0.4852941176470588, + "pass@8": 0.5176470588235295, + "pass@16": 0.5764705882352941, + "pass@32": 0.6617647058823529, + "pass@64": 0.7382352941176471, + "pass@128": 0.8058823529411765 + }, + "19": { + "pass@1": 0.40773809523809523, + "pass@2": 0.4523809523809524, + "pass@4": 0.4851190476190476, + "pass@8": 0.5178571428571429, + "pass@16": 0.5476190476190477, + "pass@32": 0.5952380952380952, + "pass@64": 0.6488095238095238, + "pass@128": 0.6845238095238095 + }, + "20": { + "pass@1": 0.3712574850299401, + "pass@2": 0.4041916167664671, + "pass@4": 0.45209580838323354, + "pass@8": 0.4820359281437126, + "pass@16": 0.5239520958083832, + "pass@32": 0.5748502994011976, + "pass@64": 0.5898203592814372, + "pass@128": 0.6287425149700598 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6596385542168675, + "12": 0.5956112852664577, + "13": 0.6078431372549019, + "14": 0.5584795321637427, + "15": 0.5393939393939394, + "16": 0.5213675213675214, + "17": 0.42857142857142855, + "18": 0.4294117647058823, + "19": 0.40773809523809523, + "20": 0.3712574850299401 + }, + "pass@2": { + "11": 0.713855421686747, + "12": 0.6520376175548589, + "13": 0.6470588235294118, + "14": 0.5964912280701754, + "15": 0.5696969696969697, + "16": 0.5555555555555556, + "17": 0.468944099378882, + "18": 0.4676470588235294, + "19": 0.4523809523809524, + "20": 0.4041916167664671 + }, + "pass@4": { + "11": 0.7981927710843374, + "12": 0.7304075235109718, + "13": 0.7563025210084033, + "14": 0.652046783625731, + "15": 0.6212121212121212, + "16": 0.5811965811965812, + "17": 0.5186335403726708, + "18": 0.4852941176470588, + "19": 0.4851190476190476, + "20": 0.45209580838323354 + }, + "pass@8": { + "11": 0.8554216867469879, + "12": 0.8432601880877743, + "13": 0.8291316526610645, + "14": 0.7309941520467836, + "15": 0.6787878787878788, + "16": 0.6381766381766382, + "17": 0.5714285714285714, + "18": 0.5176470588235295, + "19": 0.5178571428571429, + "20": 0.4820359281437126 + }, + "pass@16": { + "11": 0.9126506024096386, + "12": 0.9122257053291536, + "13": 0.8935574229691877, + "14": 0.8187134502923976, + "15": 0.7606060606060606, + "16": 0.6837606837606838, + "17": 0.6366459627329193, + "18": 0.5764705882352941, + "19": 0.5476190476190477, + "20": 0.5239520958083832 + }, + "pass@32": { + "11": 0.9457831325301205, + "12": 0.9373040752351097, + "13": 0.9243697478991597, + "14": 0.8888888888888888, + "15": 0.8333333333333334, + "16": 0.7435897435897436, + "17": 0.7204968944099379, + "18": 0.6617647058823529, + "19": 0.5952380952380952, + "20": 0.5748502994011976 + }, + "pass@64": { + "11": 0.9668674698795181, + "12": 0.9529780564263323, + "13": 0.9523809523809523, + "14": 0.935672514619883, + "15": 0.8878787878787879, + "16": 0.8176638176638177, + "17": 0.7950310559006211, + "18": 0.7382352941176471, + "19": 0.6488095238095238, + "20": 0.5898203592814372 + }, + "pass@128": { + "11": 0.9789156626506024, + "12": 0.9655172413793104, + "13": 0.969187675070028, + "14": 0.956140350877193, + "15": 0.9333333333333333, + "16": 0.8547008547008547, + "17": 0.8385093167701864, + "18": 0.8058823529411765, + "19": 0.6845238095238095, + "20": 0.6287425149700598 + } + }, + "total_pass_at_k": { + "pass@1": 0.5123401724650609, + "pass@2": 0.5530776092774309, + "pass@4": 0.6083853702051739, + "pass@8": 0.6666666666666666, + "pass@16": 0.7267320844484092, + "pass@32": 0.7826345524829021, + "pass@64": 0.8287243532560215, + "pass@128": 0.8617305976806423 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.612, + "pass@2": 0.683, + "pass@4": 0.779, + "pass@8": 0.849, + "pass@16": 0.906, + "pass@32": 0.943, + "pass@64": 0.971, + "pass@128": 0.983 + }, + "12": { + "pass@1": 0.619, + "pass@2": 0.689, + "pass@4": 0.752, + "pass@8": 0.85, + "pass@16": 0.908, + "pass@32": 0.942, + "pass@64": 0.964, + "pass@128": 0.975 + }, + "13": { + "pass@1": 0.591, + "pass@2": 0.631, + "pass@4": 0.723, + "pass@8": 0.814, + "pass@16": 0.871, + "pass@32": 0.906, + "pass@64": 0.938, + "pass@128": 0.958 + }, + "14": { + "pass@1": 0.544, + "pass@2": 0.584, + "pass@4": 0.628, + "pass@8": 0.696, + "pass@16": 0.772, + "pass@32": 0.855, + "pass@64": 0.906, + "pass@128": 0.933 + }, + "15": { + "pass@1": 0.507, + "pass@2": 0.551, + "pass@4": 0.594, + "pass@8": 0.65, + "pass@16": 0.724, + "pass@32": 0.812, + "pass@64": 0.889, + "pass@128": 0.942 + }, + "16": { + "pass@1": 0.5, + "pass@2": 0.548, + "pass@4": 0.575, + "pass@8": 0.632, + "pass@16": 0.681, + "pass@32": 0.741, + "pass@64": 0.801, + "pass@128": 0.832 + }, + "17": { + "pass@1": 0.478, + "pass@2": 0.518, + "pass@4": 0.566, + "pass@8": 0.608, + "pass@16": 0.666, + "pass@32": 0.724, + "pass@64": 0.787, + "pass@128": 0.835 + }, + "18": { + "pass@1": 0.449, + "pass@2": 0.493, + "pass@4": 0.511, + "pass@8": 0.56, + "pass@16": 0.615, + "pass@32": 0.67, + "pass@64": 0.747, + "pass@128": 0.809 + }, + "19": { + "pass@1": 0.388, + "pass@2": 0.437, + "pass@4": 0.478, + "pass@8": 0.514, + "pass@16": 0.547, + "pass@32": 0.59, + "pass@64": 0.644, + "pass@128": 0.69 + }, + "20": { + "pass@1": 0.361, + "pass@2": 0.406, + "pass@4": 0.441, + "pass@8": 0.47, + "pass@16": 0.514, + "pass@32": 0.568, + "pass@64": 0.615, + "pass@128": 0.664 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.612, + "12": 0.619, + "13": 0.591, + "14": 0.544, + "15": 0.507, + "16": 0.5, + "17": 0.478, + "18": 0.449, + "19": 0.388, + "20": 0.361 + }, + "pass@2": { + "11": 0.683, + "12": 0.689, + "13": 0.631, + "14": 0.584, + "15": 0.551, + "16": 0.548, + "17": 0.518, + "18": 0.493, + "19": 0.437, + "20": 0.406 + }, + "pass@4": { + "11": 0.779, + "12": 0.752, + "13": 0.723, + "14": 0.628, + "15": 0.594, + "16": 0.575, + "17": 0.566, + "18": 0.511, + "19": 0.478, + "20": 0.441 + }, + "pass@8": { + "11": 0.849, + "12": 0.85, + "13": 0.814, + "14": 0.696, + "15": 0.65, + "16": 0.632, + "17": 0.608, + "18": 0.56, + "19": 0.514, + "20": 0.47 + }, + "pass@16": { + "11": 0.906, + "12": 0.908, + "13": 0.871, + "14": 0.772, + "15": 0.724, + "16": 0.681, + "17": 0.666, + "18": 0.615, + "19": 0.547, + "20": 0.514 + }, + "pass@32": { + "11": 0.943, + "12": 0.942, + "13": 0.906, + "14": 0.855, + "15": 0.812, + "16": 0.741, + "17": 0.724, + "18": 0.67, + "19": 0.59, + "20": 0.568 + }, + "pass@64": { + "11": 0.971, + "12": 0.964, + "13": 0.938, + "14": 0.906, + "15": 0.889, + "16": 0.801, + "17": 0.787, + "18": 0.747, + "19": 0.644, + "20": 0.615 + }, + "pass@128": { + "11": 0.983, + "12": 0.975, + "13": 0.958, + "14": 0.933, + "15": 0.942, + "16": 0.832, + "17": 0.835, + "18": 0.809, + "19": 0.69, + "20": 0.664 + } + }, + "total_pass_at_k": { + "pass@1": 0.5049, + "pass@2": 0.554, + "pass@4": 0.6047, + "pass@8": 0.6643, + "pass@16": 0.7204, + "pass@32": 0.7751, + "pass@64": 0.8262, + "pass@128": 0.8621 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/summary.csv b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..e3cbfe1d9842e27f2200a7031e9c34fef6fd85ad --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-39008,0.9902135416666666,0.87520703125,0.9296837993421052,0.0,0.0,0.0,200.16405208333333,381.9587625,1152000,1280000,2432000 diff --git a/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/summary.json b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a04233f55b6c50a590560585a26f51c25b48f7e8 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-39008", + "id_acc": 0.9902135416666666, + "ood_acc": 0.87520703125, + "total_acc": 0.9296837993421052, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.16405208333333, + "ood_avg_resp_len": 381.9587625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 750630, + "answer_accuracy": 0.9270149976288334, + "avg_response_len": 282.90025539440404, + "resp_tokens_sum": 229072258, + "pass_at_k": { + "pass@1": 0.9270149976288334, + "pass@2": 0.9493477619236584, + "pass@4": 0.96401552496224, + "pass@8": 0.9738049633109391, + "pass@16": 0.980471830554453, + "pass@32": 0.9851893630710853, + "pass@64": 0.9886710958308421, + "pass@128": 0.9914638001896934 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9435149336283186, + "pass@2": 0.954635492880868, + "pass@4": 0.9636101311451112, + "pass@8": 0.9714900775767249, + "pass@16": 0.9780677121400383, + "pass@32": 0.9842094464543852, + "pass@64": 0.9901773717362713, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997371941896025, + "pass@2": 0.9999939801102844, + "pass@4": 0.9999999982800315, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9909474206349206, + "pass@2": 0.994433508311461, + "pass@4": 0.9964954672332624, + "pass@8": 0.9978625121315736, + "pass@16": 0.9989254195211111, + "pass@32": 0.9997055216891618, + "pass@64": 0.9999901718594374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992281062874252, + "pass@2": 0.9999447463341034, + "pass@4": 0.9999994695648073, + "pass@8": 0.9999999999730462, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997133875739645, + "pass@2": 0.9989840801612077, + "pass@4": 0.9997141855973625, + "pass@8": 0.9999752317035955, + "pass@16": 0.9999998695305009, + "pass@32": 0.999999999999429, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951923076923077, + "pass@2": 0.9986281199525542, + "pass@4": 0.9998137005710825, + "pass@8": 0.9999957663923499, + "pass@16": 0.9999999980003716, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.991637323943662, + "pass@2": 0.9929615587224134, + "pass@4": 0.9940380704172542, + "pass@8": 0.9954732852979431, + "pass@16": 0.9970755009921725, + "pass@32": 0.9986975516024429, + "pass@64": 0.9998376400088083, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9542315864022662, + "pass@2": 0.969464140327006, + "pass@4": 0.9810940441934841, + "pass@8": 0.9893951399204369, + "pass@16": 0.9945360284144443, + "pass@32": 0.9977323646041456, + "pass@64": 0.9996420458747602, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9427304025423728, + "pass@2": 0.9646079396325459, + "pass@4": 0.979820588898845, + "pass@8": 0.989370272818801, + "pass@16": 0.9949116559332181, + "pass@32": 0.9979107888230833, + "pass@64": 0.9992908315443338, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.914576802507837, + "pass@2": 0.9446635956359689, + "pass@4": 0.9665589881202152, + "pass@8": 0.979981660826714, + "pass@16": 0.9874232215331469, + "pass@32": 0.9921527270385655, + "pass@64": 0.996205173271306, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9100467289719626, + "pass@2": 0.9379875266759877, + "pass@4": 0.958589766816531, + "pass@8": 0.9731059061694559, + "pass@16": 0.9830554969430491, + "pass@32": 0.9893467916867982, + "pass@64": 0.992726298871782, + "pass@128": 0.9937694704049844 + }, + "15": { + "pass@1": 0.9557216401734104, + "pass@2": 0.9786739480451507, + "pass@4": 0.988442469889963, + "pass@8": 0.9914905405416587, + "pass@16": 0.992362600034752, + "pass@32": 0.9933207386637914, + "pass@64": 0.9949478858495289, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8498739919354839, + "pass@2": 0.8913004826009657, + "pass@4": 0.9222687491684507, + "pass@8": 0.9436176461985871, + "pass@16": 0.9585810148724633, + "pass@32": 0.9695494764061836, + "pass@64": 0.9774985888021867, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8969698338150289, + "pass@2": 0.9344089225570076, + "pass@4": 0.956744095300948, + "pass@8": 0.97040257318339, + "pass@16": 0.9791461578401852, + "pass@32": 0.9845951312505338, + "pass@64": 0.9879973643102052, + "pass@128": 0.9913294797687862 + }, + "18": { + "pass@1": 0.8265625, + "pass@2": 0.8718394966254224, + "pass@4": 0.9008116189940538, + "pass@8": 0.9218754716646158, + "pass@16": 0.9370514134261215, + "pass@32": 0.9485340476411356, + "pass@64": 0.9583897263711767, + "pass@128": 0.9685714285714285 + }, + "19": { + "pass@1": 0.7230902777777778, + "pass@2": 0.7983750236456933, + "pass@4": 0.8509800430351607, + "pass@8": 0.8887121617215229, + "pass@16": 0.9156199204084648, + "pass@32": 0.9346616186743513, + "pass@64": 0.9488236564767019, + "pass@128": 0.960960960960961 + }, + "20": { + "pass@1": 0.7268909534534534, + "pass@2": 0.8097002021706746, + "pass@4": 0.8596189023669333, + "pass@8": 0.8914002622308476, + "pass@16": 0.9137724953284576, + "pass@32": 0.9295212910338141, + "pass@64": 0.9403257272551829, + "pass@128": 0.948948948948949 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 766541, + "answer_accuracy": 0.9335310307872174, + "avg_response_len": 292.15439521628997, + "resp_tokens_sum": 239893817, + "pass_at_k": { + "pass@1": 0.9335310307872174, + "pass@2": 0.953042742465052, + "pass@4": 0.9665239741758695, + "pass@8": 0.9762235468442136, + "pass@16": 0.9828267648985132, + "pass@32": 0.9872906061783097, + "pass@64": 0.9907222191039753, + "pass@128": 0.9937646141855028 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9211320935582822, + "pass@2": 0.9365198994009952, + "pass@4": 0.9526557933134124, + "pass@8": 0.9657088477738169, + "pass@16": 0.9746686601446526, + "pass@32": 0.9816210687882091, + "pass@64": 0.9878415021526776, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9968513257575757, + "pass@2": 0.9974964954664758, + "pass@4": 0.9979660738430424, + "pass@8": 0.9986534604223928, + "pass@16": 0.9994358706064412, + "pass@32": 0.9999202169903283, + "pass@64": 0.9999995805383678, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962635869565217, + "pass@2": 0.9972030839895015, + "pass@4": 0.9974499779918814, + "pass@8": 0.9977680587306922, + "pass@16": 0.9983124881128989, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998209169054442, + "pass@2": 0.9999901292782528, + "pass@4": 0.9999999811986252, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987129696531792, + "pass@2": 0.9996746421419143, + "pass@4": 0.9999665655051787, + "pass@8": 0.9999997068622991, + "pass@16": 0.9999999999917892, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9898767605633803, + "pass@2": 0.9936876178329822, + "pass@4": 0.9957979374937288, + "pass@8": 0.9972295980502696, + "pass@16": 0.9982648879987673, + "pass@32": 0.9991210789903305, + "pass@64": 0.9998322279837888, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966785179640718, + "pass@2": 0.9970903419538876, + "pass@4": 0.9971908938029453, + "pass@8": 0.997369925503324, + "pass@16": 0.9977102880852468, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.973621046686747, + "pass@2": 0.9814229852480792, + "pass@4": 0.9863388201286587, + "pass@8": 0.9898787999691451, + "pass@16": 0.993064840318199, + "pass@32": 0.9960242380201543, + "pass@64": 0.9987056869995018, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9421532131661442, + "pass@2": 0.9639698985510823, + "pass@4": 0.9794393557388399, + "pass@8": 0.9891352473786128, + "pass@16": 0.99374911404456, + "pass@32": 0.9954805067246403, + "pass@64": 0.9964824145461987, + "pass@128": 0.9968652037617555 + }, + "13": { + "pass@1": 0.9447435224089635, + "pass@2": 0.9655367068087075, + "pass@4": 0.9791388313855723, + "pass@8": 0.9881338330670343, + "pass@16": 0.9938460585401995, + "pass@32": 0.9969037090102838, + "pass@64": 0.9985099383695012, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9357638888888888, + "pass@2": 0.9592812514389654, + "pass@4": 0.976130010064531, + "pass@8": 0.9864184088906494, + "pass@16": 0.9913667647399901, + "pass@32": 0.9940353616501455, + "pass@64": 0.9955747091064813, + "pass@128": 0.9970760233918129 + }, + "15": { + "pass@1": 0.9528645833333333, + "pass@2": 0.9747278394178001, + "pass@4": 0.9865421262683073, + "pass@8": 0.993495451097799, + "pass@16": 0.9974149003571077, + "pass@32": 0.99942427013389, + "pass@64": 0.999978739438185, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8898459757834758, + "pass@2": 0.918867913946654, + "pass@4": 0.940958512130428, + "pass@8": 0.9576799933576514, + "pass@16": 0.9685457081346035, + "pass@32": 0.9757692082156921, + "pass@64": 0.9819115581804977, + "pass@128": 0.9886039886039886 + }, + "17": { + "pass@1": 0.8886354813664596, + "pass@2": 0.9250734368122461, + "pass@4": 0.9469407821499012, + "pass@8": 0.9609358541243732, + "pass@16": 0.9696919174554715, + "pass@32": 0.9768628206172025, + "pass@64": 0.9848878654881961, + "pass@128": 0.9937888198757764 + }, + "18": { + "pass@1": 0.865234375, + "pass@2": 0.9126758626679023, + "pass@4": 0.9434961421182645, + "pass@8": 0.9639815917003957, + "pass@16": 0.9758078148025395, + "pass@32": 0.9821697623332426, + "pass@64": 0.9862873674165625, + "pass@128": 0.9882352941176471 + }, + "19": { + "pass@1": 0.7456752232142857, + "pass@2": 0.8095109937429702, + "pass@4": 0.8518101247388715, + "pass@8": 0.8841118867969516, + "pass@16": 0.9092287569498064, + "pass@32": 0.9274672406486854, + "pass@64": 0.9428158675268603, + "pass@128": 0.9583333333333334 + }, + "20": { + "pass@1": 0.6919910179640718, + "pass@2": 0.7697962834174169, + "pass@4": 0.8282520268798735, + "pass@8": 0.8748525418555514, + "pass@16": 0.9103714017754003, + "pass@32": 0.9346016347947679, + "pass@64": 0.9506273862852723, + "pass@128": 0.9640718562874252 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 743820, + "answer_accuracy": 0.9284380492091389, + "avg_response_len": 312.7123554581403, + "resp_tokens_sum": 250530129, + "pass_at_k": { + "pass@1": 0.9284380492091389, + "pass@2": 0.9505996797682092, + "pass@4": 0.9653567733704562, + "pass@8": 0.9753979459646634, + "pass@16": 0.98229150543528, + "pass@32": 0.9872608599596581, + "pass@64": 0.9909840482108517, + "pass@128": 0.9936092027480428 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9478311567164179, + "pass@2": 0.9607738864731464, + "pass@4": 0.9701930362249498, + "pass@8": 0.9777165144485844, + "pass@16": 0.9850490284905359, + "pass@32": 0.9921181247105286, + "pass@64": 0.9976074742078478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999772230320699, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9994745979532164, + "pass@2": 0.9999316491688539, + "pass@4": 0.9999986720409949, + "pass@8": 0.9999999997423706, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941865808823529, + "pass@2": 0.9964205071792498, + "pass@4": 0.9969975253093362, + "pass@8": 0.9970582244883547, + "pass@16": 0.9970588234979403, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9979544558359621, + "pass@2": 0.9992621991355971, + "pass@4": 0.9998455858435678, + "pass@8": 0.9999935023705825, + "pass@16": 0.9999999931461893, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960195806962026, + "pass@2": 0.9982615961078442, + "pass@4": 0.9994898540174567, + "pass@8": 0.9999461399811752, + "pass@16": 0.9999994164199608, + "pass@32": 0.9999999999832518, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913898273273273, + "pass@2": 0.9939999054172283, + "pass@4": 0.9949980785059523, + "pass@8": 0.9957539554064397, + "pass@16": 0.9965150597531441, + "pass@32": 0.9969397435790592, + "pass@64": 0.9969968106564769, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9915846061093248, + "pass@2": 0.9950103014152974, + "pass@4": 0.9964494795867558, + "pass@8": 0.9972464958966785, + "pass@16": 0.9978505020760328, + "pass@32": 0.9986542070682007, + "pass@64": 0.999607565131529, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9640625, + "pass@2": 0.9797177696537934, + "pass@4": 0.9911180554811603, + "pass@8": 0.9973277243916092, + "pass@16": 0.9995487596571645, + "pass@32": 0.9999809674873567, + "pass@64": 0.99999999279412, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9453363914373089, + "pass@2": 0.9667115148450481, + "pass@4": 0.9831256795308844, + "pass@8": 0.9930450861150011, + "pass@16": 0.9978774666448946, + "pass@32": 0.9996537102787295, + "pass@64": 0.999990478361063, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9026572145061729, + "pass@2": 0.9364265122241666, + "pass@4": 0.9617531060932193, + "pass@8": 0.9778678586981546, + "pass@16": 0.9868936266865624, + "pass@32": 0.9917170187367842, + "pass@64": 0.9945546397415536, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9428319732937686, + "pass@2": 0.9654668479403722, + "pass@4": 0.9797224478987603, + "pass@8": 0.9890457880872057, + "pass@16": 0.994350926646944, + "pass@32": 0.9976499071092195, + "pass@64": 0.9996521834093685, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9220196759259259, + "pass@2": 0.9517492496597652, + "pass@4": 0.9698473547288068, + "pass@8": 0.9799816784659696, + "pass@16": 0.9860164971157882, + "pass@32": 0.9894908053800399, + "pass@64": 0.9919161725258471, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8983222713864307, + "pass@2": 0.9338663972313197, + "pass@4": 0.9579245663982268, + "pass@8": 0.9727914706482771, + "pass@16": 0.9807054356298547, + "pass@32": 0.9851133092513668, + "pass@64": 0.9889024393806544, + "pass@128": 0.9941002949852508 + }, + "17": { + "pass@1": 0.8501035391566265, + "pass@2": 0.8853027760648893, + "pass@4": 0.9105946367020382, + "pass@8": 0.9327234486879625, + "pass@16": 0.9505718186178311, + "pass@32": 0.9637574221371371, + "pass@64": 0.9733407462659791, + "pass@128": 0.9819277108433735 + }, + "18": { + "pass@1": 0.8662550403225806, + "pass@2": 0.9099980949961902, + "pass@4": 0.9349534958331817, + "pass@8": 0.9524300861471281, + "pass@16": 0.9665201701946451, + "pass@32": 0.9776554085806844, + "pass@64": 0.9853613753348571, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7413378021148036, + "pass@2": 0.8133043509289437, + "pass@4": 0.8603601790146319, + "pass@8": 0.8925875144159812, + "pass@16": 0.9169766051361591, + "pass@32": 0.9370711730923081, + "pass@64": 0.9531285297469417, + "pass@128": 0.9637462235649547 + }, + "20": { + "pass@1": 0.6867961711711712, + "pass@2": 0.7743428714147219, + "pass@4": 0.8342244167677225, + "pass@8": 0.8771601288105502, + "pass@16": 0.9078914256137317, + "pass@32": 0.9314685436519544, + "pass@64": 0.9509367136317464, + "pass@128": 0.963963963963964 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_metrics.json b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..984571072c877b902276cb4a610eb737600c88b2 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-39008", + "id": { + "answer_accuracy": 0.9907135416666667, + "avg_response_len": 200.3214800347222, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9907135416666667, + "pass@2": 0.9931487724190714, + "pass@4": 0.9948384915427239, + "pass@8": 0.9960711099211672, + "pass@16": 0.9969708040223356, + "pass@32": 0.9977382880092687, + "pass@64": 0.9984862076277796, + "pass@128": 0.9991111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9406875, + "pass@2": 0.9533716781496065, + "pass@4": 0.9641378324896887, + "pass@8": 0.9724697523538208, + "pass@16": 0.9786671505215839, + "pass@32": 0.9838716874463883, + "pass@64": 0.9887411705878092, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991015625, + "pass@2": 0.9997656250000001, + "pass@4": 0.9999710234345706, + "pass@8": 0.9999993794983861, + "pass@16": 0.9999999998409674, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999828125, + "pass@2": 0.999986343503937, + "pass@4": 0.9999998719535058, + "pass@8": 0.9999999999954992, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9946171875, + "pass@2": 0.9971462844488189, + "pass@4": 0.9982416902887138, + "pass@8": 0.9988461738144168, + "pass@16": 0.999256286269019, + "pass@32": 0.9995778513781578, + "pass@64": 0.9998779518061655, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999375, + "pass@2": 0.9999986466535433, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997296875, + "pass@2": 0.9982154281496064, + "pass@4": 0.9986314955943008, + "pass@8": 0.9989550707933128, + "pass@16": 0.999211179270352, + "pass@32": 0.9994386769381023, + "pass@64": 0.9997519685023395, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9922265625, + "pass@2": 0.9952311761811025, + "pass@4": 0.9970484478815147, + "pass@8": 0.9981975738010556, + "pass@16": 0.9989309529643214, + "pass@32": 0.9994861478507381, + "pass@64": 0.9998748564944645, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9927265625, + "pass@2": 0.9946237696850395, + "pass@4": 0.9955160627109111, + "pass@8": 0.9961720390340187, + "pass@16": 0.9966716673347747, + "pass@32": 0.9972702284700308, + "pass@64": 0.9981299212592377, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9406875, + "2": 1.0, + "3": 0.9991015625, + "4": 0.999828125, + "5": 0.9946171875, + "6": 0.9999375, + "7": 0.997296875, + "8": 0.9922265625, + "9": 0.9927265625 + }, + "per_op_avg_response_len": { + "10": 281.4289296875, + "2": 115.67825, + "3": 145.008921875, + "4": 160.077546875, + "5": 173.351421875, + "6": 198.9700859375, + "7": 224.46628125, + "8": 243.9477109375, + "9": 259.964171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4289296875, + "2": 115.67825, + "3": 145.008921875, + "4": 160.077546875, + "5": 173.351421875, + "6": 198.9700859375, + "7": 224.46628125, + "8": 243.9477109375, + "9": 259.964171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375906, + "answer_accuracy": 0.9918154761904762, + "avg_response_len": 190.8410825101317, + "resp_tokens_sum": 72330297, + "pass_at_k": { + "pass@1": 0.9918154761904762, + "pass@2": 0.9939021086459936, + "pass@4": 0.9951738429162922, + "pass@8": 0.9961733185770961, + "pass@16": 0.9969716730470076, + "pass@32": 0.9975601201879171, + "pass@64": 0.9981010907143264, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9496220501474927, + "pass@2": 0.9594972330615753, + "pass@4": 0.966436561856759, + "pass@8": 0.9727340543343289, + "pass@16": 0.9784618657970596, + "pass@32": 0.9825567911391251, + "pass@64": 0.985620534223565, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996177370030581, + "pass@2": 0.9999838215463894, + "pass@4": 0.9999999635940003, + "pass@8": 0.9999999999999807, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9942460317460318, + "pass@2": 0.9975178883889514, + "pass@4": 0.9992110941489457, + "pass@8": 0.9998670726976857, + "pass@16": 0.9999959700663595, + "pass@32": 0.9999999986446861, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993528106508875, + "pass@2": 0.9999286562922239, + "pass@4": 0.9999986563255037, + "pass@8": 0.9999999997393219, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9939903846153846, + "pass@2": 0.997067355012114, + "pass@4": 0.9988529817186313, + "pass@8": 0.9997549809210944, + "pass@16": 0.9999868735089701, + "pass@32": 0.9999999812154626, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9912852112676056, + "pass@2": 0.9926787595652655, + "pass@4": 0.9935059028712959, + "pass@8": 0.9944525916293162, + "pass@16": 0.9953237078346044, + "pass@32": 0.9963063942759401, + "pass@64": 0.9978928690234709, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386269, + "answer_accuracy": 0.9887701711992136, + "avg_response_len": 198.2218575933814, + "resp_tokens_sum": 77436558, + "pass_at_k": { + "pass@1": 0.9887701711992136, + "pass@2": 0.9914905096438636, + "pass@4": 0.9936401283821339, + "pass@8": 0.995221788264208, + "pass@16": 0.9963180462453807, + "pass@32": 0.9973450965087971, + "pass@64": 0.998526454059339, + "pass@128": 0.9996723460026212 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9213238113496932, + "pass@2": 0.9363610151683497, + "pass@4": 0.9515447243749438, + "pass@8": 0.9635215251668017, + "pass@16": 0.9717501171588053, + "pass@32": 0.9789974182409743, + "pass@64": 0.9873278512974969, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99765625, + "pass@2": 0.9993058041040325, + "pass@4": 0.9999122283010079, + "pass@8": 0.9999981196920988, + "pass@16": 0.9999999995180833, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9952445652173914, + "pass@2": 0.9969466792194455, + "pass@4": 0.997354996114616, + "pass@8": 0.9976154255526752, + "pass@16": 0.9980647799459662, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998656876790831, + "pass@2": 0.9999964747422333, + "pass@4": 0.9999999986570447, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9968388728323699, + "pass@2": 0.9980816331755495, + "pass@4": 0.998801978690525, + "pass@8": 0.9995193591669413, + "pass@16": 0.9999304712731262, + "pass@32": 0.9999991345669063, + "pass@64": 0.999999999995383, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913072183098591, + "pass@2": 0.99444105578352, + "pass@4": 0.9964716870778477, + "pass@8": 0.9975479530508959, + "pass@16": 0.9981170196665599, + "pass@32": 0.9988210098675391, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941757110778443, + "pass@2": 0.9960987228299307, + "pass@4": 0.9969899691617888, + "pass@8": 0.9975118492787143, + "pass@16": 0.9980009131217787, + "pass@32": 0.998746881938581, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379127, + "answer_accuracy": 0.9916068588885169, + "avg_response_len": 211.8646687730164, + "resp_tokens_sum": 81003490, + "pass_at_k": { + "pass@1": 0.9916068588885169, + "pass@2": 0.9940963416932697, + "pass@4": 0.9957304998952468, + "pass@8": 0.9968375946439122, + "pass@16": 0.9976369049775453, + "pass@32": 0.9983166527158173, + "pass@64": 0.9988268501023745, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9504897388059701, + "pass@2": 0.9637266423786578, + "pass@4": 0.9740664414149726, + "pass@8": 0.9809101215824065, + "pass@16": 0.9856060890227171, + "pass@32": 0.9899456026975152, + "pass@64": 0.9932744178001089, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996573464912281, + "pass@2": 0.9999622271722615, + "pass@4": 0.9999996258691347, + "pass@8": 0.9999999999868394, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9943244485294118, + "pass@2": 0.9970045449282076, + "pass@4": 0.9982432999183926, + "pass@8": 0.9991491885263906, + "pass@16": 0.999780019461051, + "pass@32": 0.9999893918747133, + "pass@64": 0.9999999972066469, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9955992879746836, + "pass@2": 0.996529421658527, + "pass@4": 0.9969824845865787, + "pass@8": 0.9972195335748744, + "pass@16": 0.9975798614235771, + "pass@32": 0.9982246087909897, + "pass@64": 0.9992150902023323, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9915540540540541, + "pass@2": 0.9943531129554752, + "pass@4": 0.9959725785965943, + "pass@8": 0.9974309201820005, + "pass@16": 0.9988093286723544, + "pass@32": 0.9997137994250378, + "pass@64": 0.9999907019175951, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928155144694534, + "pass@2": 0.9952599248044157, + "pass@4": 0.9962277089760885, + "pass@8": 0.9966958564196817, + "pass@16": 0.9967827847936206, + "pass@32": 0.9967845657060646, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.885690625, + "avg_response_len": 383.00255859375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.885690625, + "pass@2": 0.9200112573818701, + "pass@4": 0.942923148293958, + "pass@8": 0.958625203138694, + "pass@16": 0.9694694341851866, + "pass@32": 0.9770299626435983, + "pass@64": 0.9823190794962792, + "pass@128": 0.9862 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9672890625, + "pass@2": 0.9792196112204721, + "pass@4": 0.9876725367454068, + "pass@8": 0.9931556909709873, + "pass@16": 0.9965690141758498, + "pass@32": 0.9987403608877776, + "pass@64": 0.9997945504249862, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.94403125, + "pass@2": 0.9650046751968498, + "pass@4": 0.9803290725534306, + "pass@8": 0.9897228707564851, + "pass@16": 0.9946531190349943, + "pass@32": 0.9972332972459661, + "pass@64": 0.9984116059795667, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.916140625, + "pass@2": 0.9453719242125976, + "pass@4": 0.9668695395575562, + "pass@8": 0.9807555169922598, + "pass@16": 0.9890228233786187, + "pass@32": 0.993949297704133, + "pass@64": 0.996369946463906, + "pass@128": 0.997 + }, + "14": { + "pass@1": 0.935296875, + "pass@2": 0.9578765994094488, + "pass@4": 0.9735225153730781, + "pass@8": 0.9836855570680625, + "pass@16": 0.9898838103039541, + "pass@32": 0.9936592530833573, + "pass@64": 0.9959260110166179, + "pass@128": 0.997 + }, + "15": { + "pass@1": 0.9557890625, + "pass@2": 0.9762531988188982, + "pass@4": 0.9863179638170224, + "pass@8": 0.9915235126224122, + "pass@16": 0.9943714637710914, + "pass@32": 0.995850686490616, + "pass@64": 0.9964835868315772, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.8906953125, + "pass@2": 0.9241689222440934, + "pass@4": 0.948281650543682, + "pass@8": 0.9639396977837194, + "pass@16": 0.9735715953792723, + "pass@32": 0.9798112769679606, + "pass@64": 0.9841122641368442, + "pass@128": 0.987 + }, + "17": { + "pass@1": 0.8818203125, + "pass@2": 0.9120867372047253, + "pass@4": 0.9315974080427447, + "pass@8": 0.9459496833791332, + "pass@16": 0.9576602114698388, + "pass@32": 0.9668353639277902, + "pass@64": 0.9742532271120506, + "pass@128": 0.982 + }, + "18": { + "pass@1": 0.865734375, + "pass@2": 0.9057982283464578, + "pass@4": 0.9292217935883013, + "pass@8": 0.9460961672645261, + "pass@16": 0.9592416811901567, + "pass@32": 0.9702689696994155, + "pass@64": 0.9794483921548097, + "pass@128": 0.986 + }, + "19": { + "pass@1": 0.755796875, + "pass@2": 0.8165334645669293, + "pass@4": 0.8603745553055865, + "pass@8": 0.8936782117684824, + "pass@16": 0.9184161327916489, + "pass@32": 0.9362162888664756, + "pass@64": 0.9488619959591215, + "pass@128": 0.958 + }, + "20": { + "pass@1": 0.7443125, + "pass@2": 0.8177992125984257, + "pass@4": 0.8650444474128233, + "pass@8": 0.8977451227808206, + "pass@16": 0.9213044903564043, + "pass@32": 0.9377348315624475, + "pass@64": 0.94952921488329, + "pass@128": 0.959 + } + }, + "per_op_accuracy": { + "11": 0.9672890625, + "12": 0.94403125, + "13": 0.916140625, + "14": 0.935296875, + "15": 0.9557890625, + "16": 0.8906953125, + "17": 0.8818203125, + "18": 0.865734375, + "19": 0.755796875, + "20": 0.7443125 + }, + "per_op_avg_response_len": { + "11": 301.0711640625, + "12": 313.659125, + "13": 334.7933046875, + "14": 361.8718984375, + "15": 381.90665625, + "16": 393.8262265625, + "17": 414.7526796875, + "18": 432.33415625, + "19": 436.709125, + "20": 459.10125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 301.0711640625, + "12": 313.659125, + "13": 334.7933046875, + "14": 361.8718984375, + "15": 381.90665625, + "16": 393.8262265625, + "17": 414.7526796875, + "18": 432.33415625, + "19": 436.709125, + "20": 459.10125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 380710, + "answer_accuracy": 0.883892087667162, + "avg_response_len": 365.21493313521546, + "resp_tokens_sum": 157305376, + "pass_at_k": { + "pass@1": 0.883892087667162, + "pass@2": 0.9177241403517014, + "pass@4": 0.9405296338607779, + "pass@8": 0.9563953781245412, + "pass@16": 0.9678041746692728, + "pass@32": 0.9762229110359666, + "pass@64": 0.9821707646357616, + "pass@128": 0.986329866270431 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9613358711048159, + "pass@2": 0.9751445010149226, + "pass@4": 0.9847932027264298, + "pass@8": 0.9914273628571794, + "pass@16": 0.9961681616814383, + "pass@32": 0.99904074927096, + "pass@64": 0.9999499987724876, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9385151836158192, + "pass@2": 0.9597627924952182, + "pass@4": 0.9762562224743094, + "pass@8": 0.9865212132430582, + "pass@16": 0.9922087450214933, + "pass@32": 0.9959870309300201, + "pass@64": 0.9983853008294012, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9068867554858934, + "pass@2": 0.9370576265643127, + "pass@4": 0.9592523463955722, + "pass@8": 0.9740163899882339, + "pass@16": 0.9836944065507172, + "pass@32": 0.9896518391446572, + "pass@64": 0.9926987166516155, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.9166179906542056, + "pass@2": 0.941457704761204, + "pass@4": 0.9603239408158093, + "pass@8": 0.9735008083823662, + "pass@16": 0.9829294702078442, + "pass@32": 0.9901209671439047, + "pass@64": 0.9952071961226423, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9691112716763006, + "pass@2": 0.9872729962222933, + "pass@4": 0.9933811493397139, + "pass@8": 0.9952925662134263, + "pass@16": 0.9963578351065164, + "pass@32": 0.9974679316498488, + "pass@64": 0.9985359192993869, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8643397177419355, + "pass@2": 0.9029233870967743, + "pass@4": 0.9338913727517938, + "pass@8": 0.9545205921905574, + "pass@16": 0.9671956269598453, + "pass@32": 0.9761510160969632, + "pass@64": 0.9823101727503963, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8985503973988439, + "pass@2": 0.9290570524782674, + "pass@4": 0.9477699960554062, + "pass@8": 0.9613079311183532, + "pass@16": 0.9720480116375539, + "pass@32": 0.9786693409502875, + "pass@64": 0.9811048717975289, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8410714285714286, + "pass@2": 0.8791718222722164, + "pass@4": 0.9039378402699658, + "pass@8": 0.9225013658392592, + "pass@16": 0.9380212263183387, + "pass@32": 0.9522564152117664, + "pass@64": 0.9651583391167925, + "pass@128": 0.9771428571428571 + }, + "19": { + "pass@1": 0.7519707207207207, + "pass@2": 0.8127601026223072, + "pass@4": 0.8581806291668495, + "pass@8": 0.8929533160295472, + "pass@16": 0.9184558230351982, + "pass@32": 0.9365729445914778, + "pass@64": 0.9498045174907821, + "pass@128": 0.960960960960961 + }, + "20": { + "pass@1": 0.7815080705705706, + "pass@2": 0.8459900599418323, + "pass@4": 0.8831622091720508, + "pass@8": 0.9091561650991454, + "pass@16": 0.9291310201543674, + "pass@32": 0.9450531167298203, + "pass@64": 0.9576555137668181, + "pass@128": 0.9669669669669669 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 384240, + "answer_accuracy": 0.8926181980374666, + "avg_response_len": 378.5435669417187, + "resp_tokens_sum": 162949378, + "pass_at_k": { + "pass@1": 0.8926181980374666, + "pass@2": 0.9252718107075417, + "pass@4": 0.9472593369796253, + "pass@8": 0.9624001560351806, + "pass@16": 0.9724805346590213, + "pass@32": 0.9790182015816211, + "pass@64": 0.9834010449645564, + "pass@128": 0.9869164436515017 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9735504518072289, + "pass@2": 0.9807889283512, + "pass@4": 0.9859694000975779, + "pass@8": 0.9903034578261026, + "pass@16": 0.9939776137045641, + "pass@32": 0.9972317237104557, + "pass@64": 0.9994343404291871, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.943255289968652, + "pass@2": 0.9643270370004688, + "pass@4": 0.9795041327122507, + "pass@8": 0.9901062501212899, + "pass@16": 0.9963812763120088, + "pass@32": 0.999159903580148, + "pass@64": 0.9999519319520963, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9459252450980392, + "pass@2": 0.969042242881405, + "pass@4": 0.9843180109314063, + "pass@8": 0.9930887640011501, + "pass@16": 0.9972316564809265, + "pass@32": 0.9991371398635513, + "pass@64": 0.9999152127956612, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9421829312865497, + "pass@2": 0.9626221681171436, + "pass@4": 0.9766211556340545, + "pass@8": 0.9860053568807828, + "pass@16": 0.9912234750641322, + "pass@32": 0.9933893500190114, + "pass@64": 0.994067684369298, + "pass@128": 0.9941520467836257 + }, + "15": { + "pass@1": 0.9615056818181819, + "pass@2": 0.9816444464328327, + "pass@4": 0.9924044471713761, + "pass@8": 0.9970935786701942, + "pass@16": 0.9988697871371285, + "pass@32": 0.9997170217125875, + "pass@64": 0.9999906185608666, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9047364672364673, + "pass@2": 0.9336404564012831, + "pass@4": 0.9554778179436971, + "pass@8": 0.9713046676762731, + "pass@16": 0.981129930647535, + "pass@32": 0.9863700316854713, + "pass@64": 0.9898460871084831, + "pass@128": 0.9943019943019943 + }, + "17": { + "pass@1": 0.8905764751552795, + "pass@2": 0.9193115891084263, + "pass@4": 0.9371282252373729, + "pass@8": 0.9495247038246433, + "pass@16": 0.9600510648716486, + "pass@32": 0.9696963337203404, + "pass@64": 0.9783483636870083, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8783318014705882, + "pass@2": 0.9212120773506257, + "pass@4": 0.9446521532234942, + "pass@8": 0.9603982535156668, + "pass@16": 0.971534461743965, + "pass@32": 0.9800680670995429, + "pass@64": 0.986866489127435, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7672526041666666, + "pass@2": 0.8213970812242222, + "pass@4": 0.8611331446292425, + "pass@8": 0.8925303692099814, + "pass@16": 0.9155570589381155, + "pass@32": 0.9306472438901101, + "pass@64": 0.9397489108469029, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.7179079341317365, + "pass@2": 0.797639710736008, + "pass@4": 0.8541305002480968, + "pass@8": 0.8923328681460105, + "pass@16": 0.9176848689576249, + "pass@32": 0.9338587403043053, + "pass@64": 0.9451756698292793, + "pass@128": 0.9550898203592815 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 368734, + "answer_accuracy": 0.8804200412591687, + "avg_response_len": 405.8787653766809, + "resp_tokens_sum": 169988521, + "pass_at_k": { + "pass@1": 0.8804200412591687, + "pass@2": 0.9169565226546439, + "pass@4": 0.9409279079265469, + "pass@8": 0.9570384655414117, + "pass@16": 0.9680871809387125, + "pass@32": 0.9758164177508913, + "pass@64": 0.9813595531013495, + "pass@128": 0.9853300733496333 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9673611111111111, + "pass@2": 0.9821323115860517, + "pass@4": 0.9926942709542262, + "pass@8": 0.9980986789972592, + "pass@16": 0.9997494773091639, + "pass@32": 0.9999937908674831, + "pass@64": 0.9999999994787548, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.950759747706422, + "pass@2": 0.9713404337932532, + "pass@4": 0.9855429708327742, + "pass@8": 0.9928148852591774, + "pass@16": 0.995613443895582, + "pass@32": 0.9967029940509836, + "pass@64": 0.9969374409579194, + "pass@128": 0.9969418960244648 + }, + "13": { + "pass@1": 0.8924334490740741, + "pass@2": 0.9274767303392636, + "pass@4": 0.9551435529470846, + "pass@8": 0.9738012340666778, + "pass@16": 0.9852240935964495, + "pass@32": 0.9924642039682082, + "pass@64": 0.9960781632221886, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9461007047477745, + "pass@2": 0.9686999842286034, + "pass@4": 0.9829498372236113, + "pass@8": 0.9910325386471672, + "pass@16": 0.995148432923748, + "pass@32": 0.9973034567467718, + "pass@64": 0.9984965964597918, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9357397762345679, + "pass@2": 0.9589940580344121, + "pass@4": 0.9725759832682954, + "pass@8": 0.981825320220379, + "pass@16": 0.987668589719086, + "pass@32": 0.9901856943660934, + "pass@64": 0.9907199216941467, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.9002581120943953, + "pass@2": 0.9337901830302185, + "pass@4": 0.9539900026324142, + "pass@8": 0.9649273623901918, + "pass@16": 0.971576240013085, + "pass@32": 0.9763674951513314, + "pass@64": 0.9798234041567667, + "pass@128": 0.9823008849557522 + }, + "17": { + "pass@1": 0.8558923192771084, + "pass@2": 0.8873935703443695, + "pass@4": 0.9093786171118668, + "pass@8": 0.9264764595802648, + "pass@16": 0.9403468570920928, + "pass@32": 0.9517275677742213, + "pass@64": 0.963140868562918, + "pass@128": 0.9759036144578314 + }, + "18": { + "pass@1": 0.8797631048387097, + "pass@2": 0.9189547879095755, + "pass@4": 0.9408445722510488, + "pass@8": 0.9570493000821247, + "pass@16": 0.9697178547928714, + "pass@32": 0.9798583276175898, + "pass@64": 0.9874463456793682, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7480173716012085, + "pass@2": 0.8153925410947498, + "pass@4": 0.8618116894187314, + "pass@8": 0.8955726691120511, + "pass@16": 0.921278465008222, + "pass@32": 0.9415106476448225, + "pass@64": 0.9571645244414844, + "pass@128": 0.9667673716012085 + }, + "20": { + "pass@1": 0.7336007882882883, + "pass@2": 0.8098284061620676, + "pass@4": 0.8578734074344312, + "pass@8": 0.8917625881142276, + "pass@16": 0.9171084516911536, + "pass@32": 0.9343042775669052, + "pass@64": 0.9457695347626426, + "pass@128": 0.954954954954955 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9354383223684211, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9354383223684211, + "pass@2": 0.9546553434521327, + "pass@4": 0.9675146266749486, + "pass@8": 0.9763627379303897, + "pass@16": 0.9824963988448785, + "pass@32": 0.9868391693957596, + "pass@64": 0.9899771928217264, + "pass@128": 0.9923157894736843 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9406875, + "pass@2": 0.9533716781496065, + "pass@4": 0.9641378324896887, + "pass@8": 0.9724697523538208, + "pass@16": 0.9786671505215839, + "pass@32": 0.9838716874463883, + "pass@64": 0.9887411705878092, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991015625, + "pass@2": 0.9997656250000001, + "pass@4": 0.9999710234345706, + "pass@8": 0.9999993794983861, + "pass@16": 0.9999999998409674, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999828125, + "pass@2": 0.999986343503937, + "pass@4": 0.9999998719535058, + "pass@8": 0.9999999999954992, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9946171875, + "pass@2": 0.9971462844488189, + "pass@4": 0.9982416902887138, + "pass@8": 0.9988461738144168, + "pass@16": 0.999256286269019, + "pass@32": 0.9995778513781578, + "pass@64": 0.9998779518061655, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999375, + "pass@2": 0.9999986466535433, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997296875, + "pass@2": 0.9982154281496064, + "pass@4": 0.9986314955943008, + "pass@8": 0.9989550707933128, + "pass@16": 0.999211179270352, + "pass@32": 0.9994386769381023, + "pass@64": 0.9997519685023395, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9922265625, + "pass@2": 0.9952311761811025, + "pass@4": 0.9970484478815147, + "pass@8": 0.9981975738010556, + "pass@16": 0.9989309529643214, + "pass@32": 0.9994861478507381, + "pass@64": 0.9998748564944645, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9927265625, + "pass@2": 0.9946237696850395, + "pass@4": 0.9955160627109111, + "pass@8": 0.9961720390340187, + "pass@16": 0.9966716673347747, + "pass@32": 0.9972702284700308, + "pass@64": 0.9981299212592377, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9672890625, + "pass@2": 0.9792196112204721, + "pass@4": 0.9876725367454068, + "pass@8": 0.9931556909709873, + "pass@16": 0.9965690141758498, + "pass@32": 0.9987403608877776, + "pass@64": 0.9997945504249862, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.94403125, + "pass@2": 0.9650046751968498, + "pass@4": 0.9803290725534306, + "pass@8": 0.9897228707564851, + "pass@16": 0.9946531190349943, + "pass@32": 0.9972332972459661, + "pass@64": 0.9984116059795667, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.916140625, + "pass@2": 0.9453719242125976, + "pass@4": 0.9668695395575562, + "pass@8": 0.9807555169922598, + "pass@16": 0.9890228233786187, + "pass@32": 0.993949297704133, + "pass@64": 0.996369946463906, + "pass@128": 0.997 + }, + "14": { + "pass@1": 0.935296875, + "pass@2": 0.9578765994094488, + "pass@4": 0.9735225153730781, + "pass@8": 0.9836855570680625, + "pass@16": 0.9898838103039541, + "pass@32": 0.9936592530833573, + "pass@64": 0.9959260110166179, + "pass@128": 0.997 + }, + "15": { + "pass@1": 0.9557890625, + "pass@2": 0.9762531988188982, + "pass@4": 0.9863179638170224, + "pass@8": 0.9915235126224122, + "pass@16": 0.9943714637710914, + "pass@32": 0.995850686490616, + "pass@64": 0.9964835868315772, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.8906953125, + "pass@2": 0.9241689222440934, + "pass@4": 0.948281650543682, + "pass@8": 0.9639396977837194, + "pass@16": 0.9735715953792723, + "pass@32": 0.9798112769679606, + "pass@64": 0.9841122641368442, + "pass@128": 0.987 + }, + "17": { + "pass@1": 0.8818203125, + "pass@2": 0.9120867372047253, + "pass@4": 0.9315974080427447, + "pass@8": 0.9459496833791332, + "pass@16": 0.9576602114698388, + "pass@32": 0.9668353639277902, + "pass@64": 0.9742532271120506, + "pass@128": 0.982 + }, + "18": { + "pass@1": 0.865734375, + "pass@2": 0.9057982283464578, + "pass@4": 0.9292217935883013, + "pass@8": 0.9460961672645261, + "pass@16": 0.9592416811901567, + "pass@32": 0.9702689696994155, + "pass@64": 0.9794483921548097, + "pass@128": 0.986 + }, + "19": { + "pass@1": 0.755796875, + "pass@2": 0.8165334645669293, + "pass@4": 0.8603745553055865, + "pass@8": 0.8936782117684824, + "pass@16": 0.9184161327916489, + "pass@32": 0.9362162888664756, + "pass@64": 0.9488619959591215, + "pass@128": 0.958 + }, + "20": { + "pass@1": 0.7443125, + "pass@2": 0.8177992125984257, + "pass@4": 0.8650444474128233, + "pass@8": 0.8977451227808206, + "pass@16": 0.9213044903564043, + "pass@32": 0.9377348315624475, + "pass@64": 0.94952921488329, + "pass@128": 0.959 + } + }, + "per_op_accuracy": { + "10": 0.9406875, + "2": 1.0, + "3": 0.9991015625, + "4": 0.999828125, + "5": 0.9946171875, + "6": 0.9999375, + "7": 0.997296875, + "8": 0.9922265625, + "9": 0.9927265625, + "11": 0.9672890625, + "12": 0.94403125, + "13": 0.916140625, + "14": 0.935296875, + "15": 0.9557890625, + "16": 0.8906953125, + "17": 0.8818203125, + "18": 0.865734375, + "19": 0.755796875, + "20": 0.7443125 + }, + "per_op_avg_response_len": { + "10": 281.4289296875, + "2": 115.67825, + "3": 145.008921875, + "4": 160.077546875, + "5": 173.351421875, + "6": 198.9700859375, + "7": 224.46628125, + "8": 243.9477109375, + "9": 259.964171875, + "11": 301.0711640625, + "12": 313.659125, + "13": 334.7933046875, + "14": 361.8718984375, + "15": 381.90665625, + "16": 393.8262265625, + "17": 414.7526796875, + "18": 432.33415625, + "19": 436.709125, + "20": 459.10125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4289296875, + "2": 115.67825, + "3": 145.008921875, + "4": 160.077546875, + "5": 173.351421875, + "6": 198.9700859375, + "7": 224.46628125, + "8": 243.9477109375, + "9": 259.964171875, + "11": 301.0711640625, + "12": 313.659125, + "13": 334.7933046875, + "14": 361.8718984375, + "15": 381.90665625, + "16": 393.8262265625, + "17": 414.7526796875, + "18": 432.33415625, + "19": 436.709125, + "20": 459.10125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 756616, + "answer_accuracy": 0.9344076035409421, + "avg_response_len": 283.59606312243125, + "resp_tokens_sum": 229635673, + "pass_at_k": { + "pass@1": 0.9344076035409421, + "pass@2": 0.9533806316762856, + "pass@4": 0.9661068553298521, + "pass@8": 0.9750141706759139, + "pass@16": 0.9814565557468075, + "pass@32": 0.9862101820285257, + "pass@64": 0.9896272451161039, + "pass@128": 0.9920961112867531 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9496220501474927, + "pass@2": 0.9594972330615753, + "pass@4": 0.966436561856759, + "pass@8": 0.9727340543343289, + "pass@16": 0.9784618657970596, + "pass@32": 0.9825567911391251, + "pass@64": 0.985620534223565, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996177370030581, + "pass@2": 0.9999838215463894, + "pass@4": 0.9999999635940003, + "pass@8": 0.9999999999999807, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9942460317460318, + "pass@2": 0.9975178883889514, + "pass@4": 0.9992110941489457, + "pass@8": 0.9998670726976857, + "pass@16": 0.9999959700663595, + "pass@32": 0.9999999986446861, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993528106508875, + "pass@2": 0.9999286562922239, + "pass@4": 0.9999986563255037, + "pass@8": 0.9999999997393219, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9939903846153846, + "pass@2": 0.997067355012114, + "pass@4": 0.9988529817186313, + "pass@8": 0.9997549809210944, + "pass@16": 0.9999868735089701, + "pass@32": 0.9999999812154626, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9912852112676056, + "pass@2": 0.9926787595652655, + "pass@4": 0.9935059028712959, + "pass@8": 0.9944525916293162, + "pass@16": 0.9953237078346044, + "pass@32": 0.9963063942759401, + "pass@64": 0.9978928690234709, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9613358711048159, + "pass@2": 0.9751445010149226, + "pass@4": 0.9847932027264298, + "pass@8": 0.9914273628571794, + "pass@16": 0.9961681616814383, + "pass@32": 0.99904074927096, + "pass@64": 0.9999499987724876, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9385151836158192, + "pass@2": 0.9597627924952182, + "pass@4": 0.9762562224743094, + "pass@8": 0.9865212132430582, + "pass@16": 0.9922087450214933, + "pass@32": 0.9959870309300201, + "pass@64": 0.9983853008294012, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9068867554858934, + "pass@2": 0.9370576265643127, + "pass@4": 0.9592523463955722, + "pass@8": 0.9740163899882339, + "pass@16": 0.9836944065507172, + "pass@32": 0.9896518391446572, + "pass@64": 0.9926987166516155, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.9166179906542056, + "pass@2": 0.941457704761204, + "pass@4": 0.9603239408158093, + "pass@8": 0.9735008083823662, + "pass@16": 0.9829294702078442, + "pass@32": 0.9901209671439047, + "pass@64": 0.9952071961226423, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9691112716763006, + "pass@2": 0.9872729962222933, + "pass@4": 0.9933811493397139, + "pass@8": 0.9952925662134263, + "pass@16": 0.9963578351065164, + "pass@32": 0.9974679316498488, + "pass@64": 0.9985359192993869, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8643397177419355, + "pass@2": 0.9029233870967743, + "pass@4": 0.9338913727517938, + "pass@8": 0.9545205921905574, + "pass@16": 0.9671956269598453, + "pass@32": 0.9761510160969632, + "pass@64": 0.9823101727503963, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8985503973988439, + "pass@2": 0.9290570524782674, + "pass@4": 0.9477699960554062, + "pass@8": 0.9613079311183532, + "pass@16": 0.9720480116375539, + "pass@32": 0.9786693409502875, + "pass@64": 0.9811048717975289, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8410714285714286, + "pass@2": 0.8791718222722164, + "pass@4": 0.9039378402699658, + "pass@8": 0.9225013658392592, + "pass@16": 0.9380212263183387, + "pass@32": 0.9522564152117664, + "pass@64": 0.9651583391167925, + "pass@128": 0.9771428571428571 + }, + "19": { + "pass@1": 0.7519707207207207, + "pass@2": 0.8127601026223072, + "pass@4": 0.8581806291668495, + "pass@8": 0.8929533160295472, + "pass@16": 0.9184558230351982, + "pass@32": 0.9365729445914778, + "pass@64": 0.9498045174907821, + "pass@128": 0.960960960960961 + }, + "20": { + "pass@1": 0.7815080705705706, + "pass@2": 0.8459900599418323, + "pass@4": 0.8831622091720508, + "pass@8": 0.9091561650991454, + "pass@16": 0.9291310201543674, + "pass@32": 0.9450531167298203, + "pass@64": 0.9576555137668181, + "pass@128": 0.9669669669669669 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 770509, + "answer_accuracy": 0.9383634547934528, + "avg_response_len": 292.753721745908, + "resp_tokens_sum": 240385936, + "pass_at_k": { + "pass@1": 0.9383634547934528, + "pass@2": 0.9567760147844829, + "pass@4": 0.9693254594052584, + "pass@8": 0.9780153737379047, + "pass@16": 0.9838214676849862, + "pass@32": 0.9877374039694216, + "pass@64": 0.9905971086523647, + "pass@128": 0.9929851909586905 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9213238113496932, + "pass@2": 0.9363610151683497, + "pass@4": 0.9515447243749438, + "pass@8": 0.9635215251668017, + "pass@16": 0.9717501171588053, + "pass@32": 0.9789974182409743, + "pass@64": 0.9873278512974969, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99765625, + "pass@2": 0.9993058041040325, + "pass@4": 0.9999122283010079, + "pass@8": 0.9999981196920988, + "pass@16": 0.9999999995180833, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9952445652173914, + "pass@2": 0.9969466792194455, + "pass@4": 0.997354996114616, + "pass@8": 0.9976154255526752, + "pass@16": 0.9980647799459662, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998656876790831, + "pass@2": 0.9999964747422333, + "pass@4": 0.9999999986570447, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9968388728323699, + "pass@2": 0.9980816331755495, + "pass@4": 0.998801978690525, + "pass@8": 0.9995193591669413, + "pass@16": 0.9999304712731262, + "pass@32": 0.9999991345669063, + "pass@64": 0.999999999995383, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913072183098591, + "pass@2": 0.99444105578352, + "pass@4": 0.9964716870778477, + "pass@8": 0.9975479530508959, + "pass@16": 0.9981170196665599, + "pass@32": 0.9988210098675391, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941757110778443, + "pass@2": 0.9960987228299307, + "pass@4": 0.9969899691617888, + "pass@8": 0.9975118492787143, + "pass@16": 0.9980009131217787, + "pass@32": 0.998746881938581, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9735504518072289, + "pass@2": 0.9807889283512, + "pass@4": 0.9859694000975779, + "pass@8": 0.9903034578261026, + "pass@16": 0.9939776137045641, + "pass@32": 0.9972317237104557, + "pass@64": 0.9994343404291871, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.943255289968652, + "pass@2": 0.9643270370004688, + "pass@4": 0.9795041327122507, + "pass@8": 0.9901062501212899, + "pass@16": 0.9963812763120088, + "pass@32": 0.999159903580148, + "pass@64": 0.9999519319520963, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9459252450980392, + "pass@2": 0.969042242881405, + "pass@4": 0.9843180109314063, + "pass@8": 0.9930887640011501, + "pass@16": 0.9972316564809265, + "pass@32": 0.9991371398635513, + "pass@64": 0.9999152127956612, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9421829312865497, + "pass@2": 0.9626221681171436, + "pass@4": 0.9766211556340545, + "pass@8": 0.9860053568807828, + "pass@16": 0.9912234750641322, + "pass@32": 0.9933893500190114, + "pass@64": 0.994067684369298, + "pass@128": 0.9941520467836257 + }, + "15": { + "pass@1": 0.9615056818181819, + "pass@2": 0.9816444464328327, + "pass@4": 0.9924044471713761, + "pass@8": 0.9970935786701942, + "pass@16": 0.9988697871371285, + "pass@32": 0.9997170217125875, + "pass@64": 0.9999906185608666, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9047364672364673, + "pass@2": 0.9336404564012831, + "pass@4": 0.9554778179436971, + "pass@8": 0.9713046676762731, + "pass@16": 0.981129930647535, + "pass@32": 0.9863700316854713, + "pass@64": 0.9898460871084831, + "pass@128": 0.9943019943019943 + }, + "17": { + "pass@1": 0.8905764751552795, + "pass@2": 0.9193115891084263, + "pass@4": 0.9371282252373729, + "pass@8": 0.9495247038246433, + "pass@16": 0.9600510648716486, + "pass@32": 0.9696963337203404, + "pass@64": 0.9783483636870083, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8783318014705882, + "pass@2": 0.9212120773506257, + "pass@4": 0.9446521532234942, + "pass@8": 0.9603982535156668, + "pass@16": 0.971534461743965, + "pass@32": 0.9800680670995429, + "pass@64": 0.986866489127435, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7672526041666666, + "pass@2": 0.8213970812242222, + "pass@4": 0.8611331446292425, + "pass@8": 0.8925303692099814, + "pass@16": 0.9155570589381155, + "pass@32": 0.9306472438901101, + "pass@64": 0.9397489108469029, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.7179079341317365, + "pass@2": 0.797639710736008, + "pass@4": 0.8541305002480968, + "pass@8": 0.8923328681460105, + "pass@16": 0.9176848689576249, + "pass@32": 0.9338587403043053, + "pass@64": 0.9451756698292793, + "pass@128": 0.9550898203592815 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 747861, + "answer_accuracy": 0.9334820358683495, + "avg_response_len": 313.2888777660169, + "resp_tokens_sum": 250992011, + "pass_at_k": { + "pass@1": 0.9334820358683495, + "pass@2": 0.9537701733126235, + "pass@4": 0.9670815015054736, + "pass@8": 0.9760319147552091, + "pass@16": 0.9821892780315375, + "pass@32": 0.9865542675416312, + "pass@64": 0.9896955198919034, + "pass@128": 0.9918517335037546 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9504897388059701, + "pass@2": 0.9637266423786578, + "pass@4": 0.9740664414149726, + "pass@8": 0.9809101215824065, + "pass@16": 0.9856060890227171, + "pass@32": 0.9899456026975152, + "pass@64": 0.9932744178001089, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996573464912281, + "pass@2": 0.9999622271722615, + "pass@4": 0.9999996258691347, + "pass@8": 0.9999999999868394, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9943244485294118, + "pass@2": 0.9970045449282076, + "pass@4": 0.9982432999183926, + "pass@8": 0.9991491885263906, + "pass@16": 0.999780019461051, + "pass@32": 0.9999893918747133, + "pass@64": 0.9999999972066469, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9955992879746836, + "pass@2": 0.996529421658527, + "pass@4": 0.9969824845865787, + "pass@8": 0.9972195335748744, + "pass@16": 0.9975798614235771, + "pass@32": 0.9982246087909897, + "pass@64": 0.9992150902023323, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9915540540540541, + "pass@2": 0.9943531129554752, + "pass@4": 0.9959725785965943, + "pass@8": 0.9974309201820005, + "pass@16": 0.9988093286723544, + "pass@32": 0.9997137994250378, + "pass@64": 0.9999907019175951, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928155144694534, + "pass@2": 0.9952599248044157, + "pass@4": 0.9962277089760885, + "pass@8": 0.9966958564196817, + "pass@16": 0.9967827847936206, + "pass@32": 0.9967845657060646, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9673611111111111, + "pass@2": 0.9821323115860517, + "pass@4": 0.9926942709542262, + "pass@8": 0.9980986789972592, + "pass@16": 0.9997494773091639, + "pass@32": 0.9999937908674831, + "pass@64": 0.9999999994787548, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.950759747706422, + "pass@2": 0.9713404337932532, + "pass@4": 0.9855429708327742, + "pass@8": 0.9928148852591774, + "pass@16": 0.995613443895582, + "pass@32": 0.9967029940509836, + "pass@64": 0.9969374409579194, + "pass@128": 0.9969418960244648 + }, + "13": { + "pass@1": 0.8924334490740741, + "pass@2": 0.9274767303392636, + "pass@4": 0.9551435529470846, + "pass@8": 0.9738012340666778, + "pass@16": 0.9852240935964495, + "pass@32": 0.9924642039682082, + "pass@64": 0.9960781632221886, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9461007047477745, + "pass@2": 0.9686999842286034, + "pass@4": 0.9829498372236113, + "pass@8": 0.9910325386471672, + "pass@16": 0.995148432923748, + "pass@32": 0.9973034567467718, + "pass@64": 0.9984965964597918, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9357397762345679, + "pass@2": 0.9589940580344121, + "pass@4": 0.9725759832682954, + "pass@8": 0.981825320220379, + "pass@16": 0.987668589719086, + "pass@32": 0.9901856943660934, + "pass@64": 0.9907199216941467, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.9002581120943953, + "pass@2": 0.9337901830302185, + "pass@4": 0.9539900026324142, + "pass@8": 0.9649273623901918, + "pass@16": 0.971576240013085, + "pass@32": 0.9763674951513314, + "pass@64": 0.9798234041567667, + "pass@128": 0.9823008849557522 + }, + "17": { + "pass@1": 0.8558923192771084, + "pass@2": 0.8873935703443695, + "pass@4": 0.9093786171118668, + "pass@8": 0.9264764595802648, + "pass@16": 0.9403468570920928, + "pass@32": 0.9517275677742213, + "pass@64": 0.963140868562918, + "pass@128": 0.9759036144578314 + }, + "18": { + "pass@1": 0.8797631048387097, + "pass@2": 0.9189547879095755, + "pass@4": 0.9408445722510488, + "pass@8": 0.9570493000821247, + "pass@16": 0.9697178547928714, + "pass@32": 0.9798583276175898, + "pass@64": 0.9874463456793682, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7480173716012085, + "pass@2": 0.8153925410947498, + "pass@4": 0.8618116894187314, + "pass@8": 0.8955726691120511, + "pass@16": 0.921278465008222, + "pass@32": 0.9415106476448225, + "pass@64": 0.9571645244414844, + "pass@128": 0.9667673716012085 + }, + "20": { + "pass@1": 0.7336007882882883, + "pass@2": 0.8098284061620676, + "pass@4": 0.8578734074344312, + "pass@8": 0.8917625881142276, + "pass@16": 0.9171084516911536, + "pass@32": 0.9343042775669052, + "pass@64": 0.9457695347626426, + "pass@128": 0.954954954954955 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..33c4103600d0ee0df2490eee79f6ff345404811f --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d1ca352d9f86e0d63954c48a8d9d5bb3c45ee26c4f708da651778b38e3fb067 +size 2725322639 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6f7230b8ae2fc6b1268d716470dc08d2ffb7a42d --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/graphs", "generated_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive", "results_dir": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1022933, "value_mismatch": 12241, "dependency_mismatch": 127157, "answer_mismatch": 10698, "extra_nodes": 13262, "missing_nodes": 9278, "perfect_with_extra": 1586, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337063, "dependency_mismatch": 41378, "missing_nodes": 2996, "extra_nodes": 4285, "perfect_with_extra": 455, "value_mismatch": 3713, "answer_mismatch": 3102}, "teachers_in_school": {"total": 390656, "perfect": 346346, "dependency_mismatch": 43663, "value_mismatch": 4718, "answer_mismatch": 4387, "extra_nodes": 4542, "missing_nodes": 3787, "perfect_with_extra": 432}, "movie_festival_awards": {"total": 382336, "perfect": 339524, "dependency_mismatch": 42116, "value_mismatch": 3810, "answer_mismatch": 3209, "extra_nodes": 4435, "perfect_with_extra": 699, "missing_nodes": 2495}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 661693, "value_mismatch": 197780, "dependency_mismatch": 580757, "answer_mismatch": 146316, "extra_nodes": 47542, "missing_nodes": 103573, "perfect_with_extra": 508, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 222428, "dependency_mismatch": 195031, "value_mismatch": 66807, "answer_mismatch": 50010, "extra_nodes": 18223, "perfect_with_extra": 196, "missing_nodes": 36102}, "movie_festival_awards": {"total": 418816, "perfect": 216477, "value_mismatch": 67144, "dependency_mismatch": 189445, "answer_mismatch": 50082, "missing_nodes": 34126, "extra_nodes": 14006, "perfect_with_extra": 136}, "teachers_in_school": {"total": 430464, "perfect": 222788, "value_mismatch": 63829, "dependency_mismatch": 196281, "answer_mismatch": 46224, "extra_nodes": 15313, "missing_nodes": 33345, "perfect_with_extra": 176}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1684626, "value_mismatch": 210021, "dependency_mismatch": 707914, "answer_mismatch": 157014, "extra_nodes": 60804, "missing_nodes": 112851, "perfect_with_extra": 2094, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 559491, "dependency_mismatch": 236409, "missing_nodes": 39098, "extra_nodes": 22508, "perfect_with_extra": 651, "value_mismatch": 70520, "answer_mismatch": 53112}, "teachers_in_school": {"total": 821120, "perfect": 569134, "dependency_mismatch": 239944, "value_mismatch": 68547, "answer_mismatch": 50611, "extra_nodes": 19855, "missing_nodes": 37132, "perfect_with_extra": 608}, "movie_festival_awards": {"total": 801152, "perfect": 556001, "dependency_mismatch": 231561, "value_mismatch": 70954, "answer_mismatch": 53291, "extra_nodes": 18441, "perfect_with_extra": 835, "missing_nodes": 36621}}}} diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ee23982da3fa9733a2be56cb163107ded2409b12 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17315f7f33be5d0bbe38c26f6925988180f9ae800ecec26ec917d3f482d71fdd +size 373300067 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..66d57a4006e077e92d953e2fa0c851b9fe8a71e9 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91a20ffbe82c14b403caaddb04fbf8534377baf5de0443e9ad56c47132d8f643 +size 421473588 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/results.json b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..ac163e74803b900e393d2fbbdfb0c0243fc4d44b --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-39008_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.996875, + "pass@4": 0.996875, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9650793650793651, + "pass@2": 0.9809523809523809, + "pass@4": 0.9873015873015873, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9820359281437125, + "pass@2": 0.9880239520958084, + "pass@4": 0.9940119760479041, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8727810650887574, + "pass@2": 0.9201183431952663, + "pass@4": 0.9704142011834319, + "pass@8": 0.9911242603550295, + "pass@16": 0.9940828402366864, + "pass@32": 0.9970414201183432, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8044871794871795, + "pass@2": 0.8557692307692307, + "pass@4": 0.9198717948717948, + "pass@8": 0.9807692307692307, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7436619718309859, + "pass@2": 0.8169014084507042, + "pass@4": 0.8845070422535212, + "pass@8": 0.9408450704225352, + "pass@16": 0.9690140845070423, + "pass@32": 0.9887323943661972, + "pass@64": 0.9887323943661972, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6489675516224189, + "pass@2": 0.7109144542772862, + "pass@4": 0.799410029498525, + "pass@8": 0.8643067846607669, + "pass@16": 0.9056047197640118, + "pass@32": 0.9262536873156342, + "pass@64": 0.9469026548672567, + "pass@128": 0.9528023598820059 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9650793650793651, + "6": 0.9820359281437125, + "7": 0.8727810650887574, + "8": 0.8044871794871795, + "9": 0.7436619718309859, + "10": 0.6489675516224189 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9809523809523809, + "6": 0.9880239520958084, + "7": 0.9201183431952663, + "8": 0.8557692307692307, + "9": 0.8169014084507042, + "10": 0.7109144542772862 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9873015873015873, + "6": 0.9940119760479041, + "7": 0.9704142011834319, + "8": 0.9198717948717948, + "9": 0.8845070422535212, + "10": 0.799410029498525 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 1.0, + "7": 0.9911242603550295, + "8": 0.9807692307692307, + "9": 0.9408450704225352, + "10": 0.8643067846607669 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 1.0, + "7": 0.9940828402366864, + "8": 1.0, + "9": 0.9690140845070423, + "10": 0.9056047197640118 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 0.9887323943661972, + "10": 0.9262536873156342 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9887323943661972, + "10": 0.9469026548672567 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9528023598820059 + } + }, + "total_pass_at_k": { + "pass@1": 0.8872002701789936, + "pass@2": 0.916919959473151, + "pass@4": 0.9490037149611618, + "pass@8": 0.9739952718676123, + "pass@16": 0.9844647078689632, + "pass@32": 0.9895305639986491, + "pass@64": 0.9922323539344816, + "pass@128": 0.994596420128335 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9970760233918129, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9529411764705882, + "pass@2": 0.9735294117647059, + "pass@4": 0.9941176470588236, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9716088328075709, + "pass@2": 0.9779179810725552, + "pass@4": 0.9905362776025236, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8607594936708861, + "pass@2": 0.9113924050632911, + "pass@4": 0.9493670886075949, + "pass@8": 0.9715189873417721, + "pass@16": 0.9968354430379747, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.8168168168168168, + "pass@2": 0.8828828828828829, + "pass@4": 0.9369369369369369, + "pass@8": 0.9669669669669669, + "pass@16": 0.987987987987988, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.77491961414791, + "pass@2": 0.842443729903537, + "pass@4": 0.8842443729903537, + "pass@8": 0.9421221864951769, + "pass@16": 0.9742765273311897, + "pass@32": 0.9903536977491961, + "pass@64": 0.9903536977491961, + "pass@128": 0.9935691318327974 + }, + "10": { + "pass@1": 0.6119402985074627, + "pass@2": 0.6865671641791045, + "pass@4": 0.755223880597015, + "pass@8": 0.8507462686567164, + "pass@16": 0.9014925373134328, + "pass@32": 0.9373134328358209, + "pass@64": 0.9492537313432836, + "pass@128": 0.9582089552238806 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9529411764705882, + "6": 0.9716088328075709, + "7": 0.8607594936708861, + "8": 0.8168168168168168, + "9": 0.77491961414791, + "10": 0.6119402985074627 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9735294117647059, + "6": 0.9779179810725552, + "7": 0.9113924050632911, + "8": 0.8828828828828829, + "9": 0.842443729903537, + "10": 0.6865671641791045 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 0.9905362776025236, + "7": 0.9493670886075949, + "8": 0.9369369369369369, + "9": 0.8842443729903537, + "10": 0.755223880597015 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.9715189873417721, + "8": 0.9669669669669669, + "9": 0.9421221864951769, + "10": 0.8507462686567164 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.987987987987988, + "9": 0.9742765273311897, + "10": 0.9014925373134328 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9903536977491961, + "10": 0.9373134328358209 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9903536977491961, + "10": 0.9492537313432836 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9935691318327974, + "10": 0.9582089552238806 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885169065952461, + "pass@2": 0.9203213927017074, + "pass@4": 0.9464345497154335, + "pass@8": 0.9698694342149313, + "pass@16": 0.9842651489789086, + "pass@32": 0.9909608302644795, + "pass@64": 0.9922999665215936, + "pass@128": 0.9936391027787077 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9797101449275363, + "pass@4": 0.9971014492753624, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9742120343839542, + "pass@2": 0.9828080229226361, + "pass@4": 0.9914040114613181, + "pass@8": 0.997134670487106, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8352601156069365, + "pass@2": 0.9075144508670521, + "pass@4": 0.9566473988439307, + "pass@8": 0.9884393063583815, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8084507042253521, + "pass@2": 0.8704225352112676, + "pass@4": 0.9295774647887324, + "pass@8": 0.9605633802816902, + "pass@16": 0.9943661971830986, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7395209580838323, + "pass@2": 0.7934131736526946, + "pass@4": 0.8772455089820359, + "pass@8": 0.9491017964071856, + "pass@16": 0.9790419161676647, + "pass@32": 0.9880239520958084, + "pass@64": 0.9970059880239521, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6411042944785276, + "pass@2": 0.7024539877300614, + "pass@4": 0.7668711656441718, + "pass@8": 0.8374233128834356, + "pass@16": 0.8773006134969326, + "pass@32": 0.9202453987730062, + "pass@64": 0.9355828220858896, + "pass@128": 0.941717791411043 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9536231884057971, + "6": 0.9742120343839542, + "7": 0.8352601156069365, + "8": 0.8084507042253521, + "9": 0.7395209580838323, + "10": 0.6411042944785276 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9797101449275363, + "6": 0.9828080229226361, + "7": 0.9075144508670521, + "8": 0.8704225352112676, + "9": 0.7934131736526946, + "10": 0.7024539877300614 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.9566473988439307, + "8": 0.9295774647887324, + "9": 0.8772455089820359, + "10": 0.7668711656441718 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9884393063583815, + "8": 0.9605633802816902, + "9": 0.9491017964071856, + "10": 0.8374233128834356 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9790419161676647, + "10": 0.8773006134969326 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9880239520958084, + "10": 0.9202453987730062 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9355828220858896 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.941717791411043 + } + }, + "total_pass_at_k": { + "pass@1": 0.8820445609436435, + "pass@2": 0.9157929226736566, + "pass@4": 0.9472477064220184, + "pass@8": 0.9705111402359109, + "pass@16": 0.9832896461336829, + "pass@32": 0.9898427260812582, + "pass@64": 0.9924639580602883, + "pass@128": 0.9937745740498034 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.988, + "pass@2": 0.998, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.957, + "pass@2": 0.978, + "pass@4": 0.993, + "pass@8": 0.998, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.976, + "pass@2": 0.983, + "pass@4": 0.992, + "pass@8": 0.998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.856, + "pass@2": 0.913, + "pass@4": 0.959, + "pass@8": 0.984, + "pass@16": 0.996, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.81, + "pass@2": 0.87, + "pass@4": 0.929, + "pass@8": 0.969, + "pass@16": 0.994, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.752, + "pass@2": 0.817, + "pass@4": 0.882, + "pass@8": 0.944, + "pass@16": 0.974, + "pass@32": 0.989, + "pass@64": 0.992, + "pass@128": 0.998 + }, + "10": { + "pass@1": 0.634, + "pass@2": 0.7, + "pass@4": 0.774, + "pass@8": 0.851, + "pass@16": 0.895, + "pass@32": 0.928, + "pass@64": 0.944, + "pass@128": 0.951 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.988, + "5": 0.957, + "6": 0.976, + "7": 0.856, + "8": 0.81, + "9": 0.752, + "10": 0.634 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.978, + "6": 0.983, + "7": 0.913, + "8": 0.87, + "9": 0.817, + "10": 0.7 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.993, + "6": 0.992, + "7": 0.959, + "8": 0.929, + "9": 0.882, + "10": 0.774 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 0.998, + "7": 0.984, + "8": 0.969, + "9": 0.944, + "10": 0.851 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 1.0, + "7": 0.996, + "8": 0.994, + "9": 0.974, + "10": 0.895 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 1.0, + "7": 0.998, + "8": 0.999, + "9": 0.989, + "10": 0.928 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 1.0, + "7": 0.999, + "8": 0.999, + "9": 0.992, + "10": 0.944 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 0.999, + "9": 0.998, + "10": 0.951 + } + }, + "total_pass_at_k": { + "pass@1": 0.8858888888888888, + "pass@2": 0.9176666666666666, + "pass@4": 0.9475555555555556, + "pass@8": 0.9714444444444444, + "pass@16": 0.984, + "pass@32": 0.9901111111111112, + "pass@64": 0.9923333333333333, + "pass@128": 0.994 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-39008_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6062322946175638, + "pass@2": 0.6968838526912181, + "pass@4": 0.7620396600566572, + "pass@8": 0.8526912181303116, + "pass@16": 0.9036827195467422, + "pass@32": 0.9546742209631728, + "pass@64": 0.9745042492917847, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6610169491525424, + "pass@2": 0.7146892655367232, + "pass@4": 0.8050847457627118, + "pass@8": 0.8587570621468926, + "pass@16": 0.9067796610169492, + "pass@32": 0.9293785310734464, + "pass@64": 0.9519774011299436, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.5705329153605015, + "pass@2": 0.6394984326018809, + "pass@4": 0.6990595611285266, + "pass@8": 0.7962382445141066, + "pass@16": 0.8432601880877743, + "pass@32": 0.8871473354231975, + "pass@64": 0.9153605015673981, + "pass@128": 0.9310344827586207 + }, + "14": { + "pass@1": 0.5451713395638629, + "pass@2": 0.5825545171339563, + "pass@4": 0.6074766355140186, + "pass@8": 0.7040498442367601, + "pass@16": 0.7538940809968847, + "pass@32": 0.8161993769470405, + "pass@64": 0.8691588785046729, + "pass@128": 0.897196261682243 + }, + "15": { + "pass@1": 0.4913294797687861, + "pass@2": 0.5202312138728323, + "pass@4": 0.5809248554913294, + "pass@8": 0.638728323699422, + "pass@16": 0.7225433526011561, + "pass@32": 0.8121387283236994, + "pass@64": 0.8872832369942196, + "pass@128": 0.9479768786127167 + }, + "16": { + "pass@1": 0.4935483870967742, + "pass@2": 0.535483870967742, + "pass@4": 0.5612903225806452, + "pass@8": 0.6290322580645161, + "pass@16": 0.6645161290322581, + "pass@32": 0.7064516129032258, + "pass@64": 0.7580645161290323, + "pass@128": 0.8032258064516129 + }, + "17": { + "pass@1": 0.5260115606936416, + "pass@2": 0.5520231213872833, + "pass@4": 0.5809248554913294, + "pass@8": 0.630057803468208, + "pass@16": 0.684971098265896, + "pass@32": 0.7427745664739884, + "pass@64": 0.8005780346820809, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.44857142857142857, + "pass@2": 0.47714285714285715, + "pass@4": 0.49714285714285716, + "pass@8": 0.5542857142857143, + "pass@16": 0.6142857142857143, + "pass@32": 0.6628571428571428, + "pass@64": 0.7114285714285714, + "pass@128": 0.7742857142857142 + }, + "19": { + "pass@1": 0.4024024024024024, + "pass@2": 0.44744744744744747, + "pass@4": 0.48348348348348347, + "pass@8": 0.5045045045045045, + "pass@16": 0.5555555555555556, + "pass@32": 0.6066066066066066, + "pass@64": 0.6516516516516516, + "pass@128": 0.6846846846846847 + }, + "20": { + "pass@1": 0.4264264264264264, + "pass@2": 0.45645645645645644, + "pass@4": 0.4774774774774775, + "pass@8": 0.5165165165165165, + "pass@16": 0.5585585585585585, + "pass@32": 0.6306306306306306, + "pass@64": 0.6696696696696697, + "pass@128": 0.7507507507507507 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6062322946175638, + "12": 0.6610169491525424, + "13": 0.5705329153605015, + "14": 0.5451713395638629, + "15": 0.4913294797687861, + "16": 0.4935483870967742, + "17": 0.5260115606936416, + "18": 0.44857142857142857, + "19": 0.4024024024024024, + "20": 0.4264264264264264 + }, + "pass@2": { + "11": 0.6968838526912181, + "12": 0.7146892655367232, + "13": 0.6394984326018809, + "14": 0.5825545171339563, + "15": 0.5202312138728323, + "16": 0.535483870967742, + "17": 0.5520231213872833, + "18": 0.47714285714285715, + "19": 0.44744744744744747, + "20": 0.45645645645645644 + }, + "pass@4": { + "11": 0.7620396600566572, + "12": 0.8050847457627118, + "13": 0.6990595611285266, + "14": 0.6074766355140186, + "15": 0.5809248554913294, + "16": 0.5612903225806452, + "17": 0.5809248554913294, + "18": 0.49714285714285716, + "19": 0.48348348348348347, + "20": 0.4774774774774775 + }, + "pass@8": { + "11": 0.8526912181303116, + "12": 0.8587570621468926, + "13": 0.7962382445141066, + "14": 0.7040498442367601, + "15": 0.638728323699422, + "16": 0.6290322580645161, + "17": 0.630057803468208, + "18": 0.5542857142857143, + "19": 0.5045045045045045, + "20": 0.5165165165165165 + }, + "pass@16": { + "11": 0.9036827195467422, + "12": 0.9067796610169492, + "13": 0.8432601880877743, + "14": 0.7538940809968847, + "15": 0.7225433526011561, + "16": 0.6645161290322581, + "17": 0.684971098265896, + "18": 0.6142857142857143, + "19": 0.5555555555555556, + "20": 0.5585585585585585 + }, + "pass@32": { + "11": 0.9546742209631728, + "12": 0.9293785310734464, + "13": 0.8871473354231975, + "14": 0.8161993769470405, + "15": 0.8121387283236994, + "16": 0.7064516129032258, + "17": 0.7427745664739884, + "18": 0.6628571428571428, + "19": 0.6066066066066066, + "20": 0.6306306306306306 + }, + "pass@64": { + "11": 0.9745042492917847, + "12": 0.9519774011299436, + "13": 0.9153605015673981, + "14": 0.8691588785046729, + "15": 0.8872832369942196, + "16": 0.7580645161290323, + "17": 0.8005780346820809, + "18": 0.7114285714285714, + "19": 0.6516516516516516, + "20": 0.6696696696696697 + }, + "pass@128": { + "11": 0.9858356940509915, + "12": 0.96045197740113, + "13": 0.9310344827586207, + "14": 0.897196261682243, + "15": 0.9479768786127167, + "16": 0.8032258064516129, + "17": 0.8352601156069365, + "18": 0.7742857142857142, + "19": 0.6846846846846847, + "20": 0.7507507507507507 + } + }, + "total_pass_at_k": { + "pass@1": 0.5179791976225855, + "pass@2": 0.563150074294205, + "pass@4": 0.6068350668647845, + "pass@8": 0.6695393759286775, + "pass@16": 0.7221396731054978, + "pass@32": 0.7762258543833581, + "pass@64": 0.8202080237741456, + "pass@128": 0.8582466567607726 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5523809523809524, + "pass@2": 0.6349206349206349, + "pass@4": 0.7428571428571429, + "pass@8": 0.8380952380952381, + "pass@16": 0.9333333333333333, + "pass@32": 0.9682539682539683, + "pass@64": 0.9841269841269841, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.6238532110091743, + "pass@2": 0.6758409785932722, + "pass@4": 0.7584097859327217, + "pass@8": 0.8440366972477065, + "pass@16": 0.9143730886850153, + "pass@32": 0.9541284403669725, + "pass@64": 0.9724770642201835, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.5709876543209876, + "pass@2": 0.6265432098765432, + "pass@4": 0.7037037037037037, + "pass@8": 0.7716049382716049, + "pass@16": 0.845679012345679, + "pass@32": 0.8858024691358025, + "pass@64": 0.9290123456790124, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.5519287833827893, + "pass@2": 0.5816023738872403, + "pass@4": 0.629080118694362, + "pass@8": 0.7062314540059347, + "pass@16": 0.7804154302670623, + "pass@32": 0.8724035608308606, + "pass@64": 0.9109792284866469, + "pass@128": 0.9376854599406528 + }, + "15": { + "pass@1": 0.4783950617283951, + "pass@2": 0.5493827160493827, + "pass@4": 0.5895061728395061, + "pass@8": 0.6419753086419753, + "pass@16": 0.7067901234567902, + "pass@32": 0.7962962962962963, + "pass@64": 0.8858024691358025, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.5309734513274337, + "pass@2": 0.5722713864306784, + "pass@4": 0.6106194690265486, + "pass@8": 0.6696165191740413, + "pass@16": 0.7109144542772862, + "pass@32": 0.7610619469026548, + "pass@64": 0.7935103244837758, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.5, + "pass@2": 0.5180722891566265, + "pass@4": 0.536144578313253, + "pass@8": 0.5783132530120482, + "pass@16": 0.6445783132530121, + "pass@32": 0.713855421686747, + "pass@64": 0.7650602409638554, + "pass@128": 0.8283132530120482 + }, + "18": { + "pass@1": 0.4870967741935484, + "pass@2": 0.5290322580645161, + "pass@4": 0.567741935483871, + "pass@8": 0.5967741935483871, + "pass@16": 0.6612903225806451, + "pass@32": 0.7161290322580646, + "pass@64": 0.7870967741935484, + "pass@128": 0.8387096774193549 + }, + "19": { + "pass@1": 0.38972809667673713, + "pass@2": 0.4350453172205438, + "pass@4": 0.47129909365558914, + "pass@8": 0.5045317220543807, + "pass@16": 0.5347432024169184, + "pass@32": 0.5830815709969789, + "pass@64": 0.6404833836858006, + "pass@128": 0.7039274924471299 + }, + "20": { + "pass@1": 0.37537537537537535, + "pass@2": 0.4144144144144144, + "pass@4": 0.44744744744744747, + "pass@8": 0.4744744744744745, + "pass@16": 0.5225225225225225, + "pass@32": 0.5975975975975976, + "pass@64": 0.6396396396396397, + "pass@128": 0.7087087087087087 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5523809523809524, + "12": 0.6238532110091743, + "13": 0.5709876543209876, + "14": 0.5519287833827893, + "15": 0.4783950617283951, + "16": 0.5309734513274337, + "17": 0.5, + "18": 0.4870967741935484, + "19": 0.38972809667673713, + "20": 0.37537537537537535 + }, + "pass@2": { + "11": 0.6349206349206349, + "12": 0.6758409785932722, + "13": 0.6265432098765432, + "14": 0.5816023738872403, + "15": 0.5493827160493827, + "16": 0.5722713864306784, + "17": 0.5180722891566265, + "18": 0.5290322580645161, + "19": 0.4350453172205438, + "20": 0.4144144144144144 + }, + "pass@4": { + "11": 0.7428571428571429, + "12": 0.7584097859327217, + "13": 0.7037037037037037, + "14": 0.629080118694362, + "15": 0.5895061728395061, + "16": 0.6106194690265486, + "17": 0.536144578313253, + "18": 0.567741935483871, + "19": 0.47129909365558914, + "20": 0.44744744744744747 + }, + "pass@8": { + "11": 0.8380952380952381, + "12": 0.8440366972477065, + "13": 0.7716049382716049, + "14": 0.7062314540059347, + "15": 0.6419753086419753, + "16": 0.6696165191740413, + "17": 0.5783132530120482, + "18": 0.5967741935483871, + "19": 0.5045317220543807, + "20": 0.4744744744744745 + }, + "pass@16": { + "11": 0.9333333333333333, + "12": 0.9143730886850153, + "13": 0.845679012345679, + "14": 0.7804154302670623, + "15": 0.7067901234567902, + "16": 0.7109144542772862, + "17": 0.6445783132530121, + "18": 0.6612903225806451, + "19": 0.5347432024169184, + "20": 0.5225225225225225 + }, + "pass@32": { + "11": 0.9682539682539683, + "12": 0.9541284403669725, + "13": 0.8858024691358025, + "14": 0.8724035608308606, + "15": 0.7962962962962963, + "16": 0.7610619469026548, + "17": 0.713855421686747, + "18": 0.7161290322580646, + "19": 0.5830815709969789, + "20": 0.5975975975975976 + }, + "pass@64": { + "11": 0.9841269841269841, + "12": 0.9724770642201835, + "13": 0.9290123456790124, + "14": 0.9109792284866469, + "15": 0.8858024691358025, + "16": 0.7935103244837758, + "17": 0.7650602409638554, + "18": 0.7870967741935484, + "19": 0.6404833836858006, + "20": 0.6396396396396397 + }, + "pass@128": { + "11": 0.9873015873015873, + "12": 0.9785932721712538, + "13": 0.9506172839506173, + "14": 0.9376854599406528, + "15": 0.9444444444444444, + "16": 0.8259587020648967, + "17": 0.8283132530120482, + "18": 0.8387096774193549, + "19": 0.7039274924471299, + "20": 0.7087087087087087 + } + }, + "total_pass_at_k": { + "pass@1": 0.5058068459657702, + "pass@2": 0.5531784841075794, + "pass@4": 0.6048288508557457, + "pass@8": 0.6616748166259169, + "pass@16": 0.7243276283618582, + "pass@32": 0.7839242053789731, + "pass@64": 0.8297677261613692, + "pass@128": 0.8694987775061125 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6475903614457831, + "pass@2": 0.7078313253012049, + "pass@4": 0.8162650602409639, + "pass@8": 0.8765060240963856, + "pass@16": 0.9337349397590361, + "pass@32": 0.9578313253012049, + "pass@64": 0.9728915662650602, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.5956112852664577, + "pass@2": 0.6394984326018809, + "pass@4": 0.7241379310344828, + "pass@8": 0.7962382445141066, + "pass@16": 0.8683385579937304, + "pass@32": 0.9153605015673981, + "pass@64": 0.9467084639498433, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.5994397759103641, + "pass@2": 0.6582633053221288, + "pass@4": 0.7591036414565826, + "pass@8": 0.8431372549019608, + "pass@16": 0.8879551820728291, + "pass@32": 0.927170868347339, + "pass@64": 0.9467787114845938, + "pass@128": 0.9635854341736695 + }, + "14": { + "pass@1": 0.5701754385964912, + "pass@2": 0.6198830409356725, + "pass@4": 0.6695906432748538, + "pass@8": 0.7280701754385965, + "pass@16": 0.8011695906432749, + "pass@32": 0.868421052631579, + "pass@64": 0.9181286549707602, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.5575757575757576, + "pass@2": 0.5909090909090909, + "pass@4": 0.6333333333333333, + "pass@8": 0.6909090909090909, + "pass@16": 0.7575757575757576, + "pass@32": 0.8272727272727273, + "pass@64": 0.8848484848484849, + "pass@128": 0.9333333333333333 + }, + "16": { + "pass@1": 0.5441595441595442, + "pass@2": 0.5811965811965812, + "pass@4": 0.6182336182336182, + "pass@8": 0.6524216524216524, + "pass@16": 0.7236467236467237, + "pass@32": 0.7720797720797721, + "pass@64": 0.8233618233618234, + "pass@128": 0.8518518518518519 + }, + "17": { + "pass@1": 0.4472049689440994, + "pass@2": 0.4720496894409938, + "pass@4": 0.5093167701863354, + "pass@8": 0.562111801242236, + "pass@16": 0.6366459627329193, + "pass@32": 0.7204968944099379, + "pass@64": 0.782608695652174, + "pass@128": 0.8198757763975155 + }, + "18": { + "pass@1": 0.4264705882352941, + "pass@2": 0.45, + "pass@4": 0.4823529411764706, + "pass@8": 0.5176470588235295, + "pass@16": 0.5558823529411765, + "pass@32": 0.6088235294117647, + "pass@64": 0.6911764705882353, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.41369047619047616, + "pass@2": 0.44642857142857145, + "pass@4": 0.4791666666666667, + "pass@8": 0.5119047619047619, + "pass@16": 0.5416666666666666, + "pass@32": 0.5863095238095238, + "pass@64": 0.6547619047619048, + "pass@128": 0.6904761904761905 + }, + "20": { + "pass@1": 0.38622754491017963, + "pass@2": 0.4251497005988024, + "pass@4": 0.46706586826347307, + "pass@8": 0.49101796407185627, + "pass@16": 0.5149700598802395, + "pass@32": 0.5508982035928144, + "pass@64": 0.6047904191616766, + "pass@128": 0.6467065868263473 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6475903614457831, + "12": 0.5956112852664577, + "13": 0.5994397759103641, + "14": 0.5701754385964912, + "15": 0.5575757575757576, + "16": 0.5441595441595442, + "17": 0.4472049689440994, + "18": 0.4264705882352941, + "19": 0.41369047619047616, + "20": 0.38622754491017963 + }, + "pass@2": { + "11": 0.7078313253012049, + "12": 0.6394984326018809, + "13": 0.6582633053221288, + "14": 0.6198830409356725, + "15": 0.5909090909090909, + "16": 0.5811965811965812, + "17": 0.4720496894409938, + "18": 0.45, + "19": 0.44642857142857145, + "20": 0.4251497005988024 + }, + "pass@4": { + "11": 0.8162650602409639, + "12": 0.7241379310344828, + "13": 0.7591036414565826, + "14": 0.6695906432748538, + "15": 0.6333333333333333, + "16": 0.6182336182336182, + "17": 0.5093167701863354, + "18": 0.4823529411764706, + "19": 0.4791666666666667, + "20": 0.46706586826347307 + }, + "pass@8": { + "11": 0.8765060240963856, + "12": 0.7962382445141066, + "13": 0.8431372549019608, + "14": 0.7280701754385965, + "15": 0.6909090909090909, + "16": 0.6524216524216524, + "17": 0.562111801242236, + "18": 0.5176470588235295, + "19": 0.5119047619047619, + "20": 0.49101796407185627 + }, + "pass@16": { + "11": 0.9337349397590361, + "12": 0.8683385579937304, + "13": 0.8879551820728291, + "14": 0.8011695906432749, + "15": 0.7575757575757576, + "16": 0.7236467236467237, + "17": 0.6366459627329193, + "18": 0.5558823529411765, + "19": 0.5416666666666666, + "20": 0.5149700598802395 + }, + "pass@32": { + "11": 0.9578313253012049, + "12": 0.9153605015673981, + "13": 0.927170868347339, + "14": 0.868421052631579, + "15": 0.8272727272727273, + "16": 0.7720797720797721, + "17": 0.7204968944099379, + "18": 0.6088235294117647, + "19": 0.5863095238095238, + "20": 0.5508982035928144 + }, + "pass@64": { + "11": 0.9728915662650602, + "12": 0.9467084639498433, + "13": 0.9467787114845938, + "14": 0.9181286549707602, + "15": 0.8848484848484849, + "16": 0.8233618233618234, + "17": 0.782608695652174, + "18": 0.6911764705882353, + "19": 0.6547619047619048, + "20": 0.6047904191616766 + }, + "pass@128": { + "11": 0.9789156626506024, + "12": 0.9717868338557993, + "13": 0.9635854341736695, + "14": 0.9502923976608187, + "15": 0.9333333333333333, + "16": 0.8518518518518519, + "17": 0.8198757763975155, + "18": 0.7705882352941177, + "19": 0.6904761904761905, + "20": 0.6467065868263473 + } + }, + "total_pass_at_k": { + "pass@1": 0.5191793041926851, + "pass@2": 0.55961938745168, + "pass@4": 0.6164139161462979, + "pass@8": 0.6675587273267916, + "pass@16": 0.7225691347011597, + "pass@32": 0.7737139458816533, + "pass@64": 0.8227772821885222, + "pass@128": 0.8578650014867678 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.603, + "pass@2": 0.681, + "pass@4": 0.774, + "pass@8": 0.856, + "pass@16": 0.923, + "pass@32": 0.96, + "pass@64": 0.977, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.628, + "pass@2": 0.678, + "pass@4": 0.764, + "pass@8": 0.834, + "pass@16": 0.897, + "pass@32": 0.933, + "pass@64": 0.957, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.581, + "pass@2": 0.642, + "pass@4": 0.722, + "pass@8": 0.805, + "pass@16": 0.86, + "pass@32": 0.901, + "pass@64": 0.931, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.556, + "pass@2": 0.595, + "pass@4": 0.636, + "pass@8": 0.713, + "pass@16": 0.779, + "pass@32": 0.853, + "pass@64": 0.9, + "pass@128": 0.929 + }, + "15": { + "pass@1": 0.509, + "pass@2": 0.553, + "pass@4": 0.601, + "pass@8": 0.657, + "pass@16": 0.729, + "pass@32": 0.812, + "pass@64": 0.886, + "pass@128": 0.942 + }, + "16": { + "pass@1": 0.524, + "pass@2": 0.564, + "pass@4": 0.598, + "pass@8": 0.651, + "pass@16": 0.701, + "pass@32": 0.748, + "pass@64": 0.793, + "pass@128": 0.828 + }, + "17": { + "pass@1": 0.492, + "pass@2": 0.515, + "pass@4": 0.543, + "pass@8": 0.591, + "pass@16": 0.656, + "pass@32": 0.726, + "pass@64": 0.783, + "pass@128": 0.828 + }, + "18": { + "pass@1": 0.453, + "pass@2": 0.484, + "pass@4": 0.514, + "pass@8": 0.555, + "pass@16": 0.609, + "pass@32": 0.661, + "pass@64": 0.728, + "pass@128": 0.793 + }, + "19": { + "pass@1": 0.402, + "pass@2": 0.443, + "pass@4": 0.478, + "pass@8": 0.507, + "pass@16": 0.544, + "pass@32": 0.592, + "pass@64": 0.649, + "pass@128": 0.693 + }, + "20": { + "pass@1": 0.396, + "pass@2": 0.432, + "pass@4": 0.464, + "pass@8": 0.494, + "pass@16": 0.532, + "pass@32": 0.593, + "pass@64": 0.638, + "pass@128": 0.702 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.603, + "12": 0.628, + "13": 0.581, + "14": 0.556, + "15": 0.509, + "16": 0.524, + "17": 0.492, + "18": 0.453, + "19": 0.402, + "20": 0.396 + }, + "pass@2": { + "11": 0.681, + "12": 0.678, + "13": 0.642, + "14": 0.595, + "15": 0.553, + "16": 0.564, + "17": 0.515, + "18": 0.484, + "19": 0.443, + "20": 0.432 + }, + "pass@4": { + "11": 0.774, + "12": 0.764, + "13": 0.722, + "14": 0.636, + "15": 0.601, + "16": 0.598, + "17": 0.543, + "18": 0.514, + "19": 0.478, + "20": 0.464 + }, + "pass@8": { + "11": 0.856, + "12": 0.834, + "13": 0.805, + "14": 0.713, + "15": 0.657, + "16": 0.651, + "17": 0.591, + "18": 0.555, + "19": 0.507, + "20": 0.494 + }, + "pass@16": { + "11": 0.923, + "12": 0.897, + "13": 0.86, + "14": 0.779, + "15": 0.729, + "16": 0.701, + "17": 0.656, + "18": 0.609, + "19": 0.544, + "20": 0.532 + }, + "pass@32": { + "11": 0.96, + "12": 0.933, + "13": 0.901, + "14": 0.853, + "15": 0.812, + "16": 0.748, + "17": 0.726, + "18": 0.661, + "19": 0.592, + "20": 0.593 + }, + "pass@64": { + "11": 0.977, + "12": 0.957, + "13": 0.931, + "14": 0.9, + "15": 0.886, + "16": 0.793, + "17": 0.783, + "18": 0.728, + "19": 0.649, + "20": 0.638 + }, + "pass@128": { + "11": 0.984, + "12": 0.97, + "13": 0.949, + "14": 0.929, + "15": 0.942, + "16": 0.828, + "17": 0.828, + "18": 0.793, + "19": 0.693, + "20": 0.702 + } + }, + "total_pass_at_k": { + "pass@1": 0.5144, + "pass@2": 0.5587, + "pass@4": 0.6094, + "pass@8": 0.6663, + "pass@16": 0.723, + "pass@32": 0.7779, + "pass@64": 0.8242, + "pass@128": 0.8618 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/summary.csv b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..b8ea0058d8a702b1bb55431c4014136f3885b8a7 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-39008,0.9907135416666667,0.885690625,0.9354383223684211,0.0,0.0,0.0,200.3214800347222,383.00255859375,1152000,1280000,2432000 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/summary.json b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6fff6102df7ea7ba5c9a7ac02742c22be66897d7 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-39008", + "id_acc": 0.9907135416666667, + "ood_acc": 0.885690625, + "total_acc": 0.9354383223684211, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.3214800347222, + "ood_avg_resp_len": 383.00255859375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 756616, + "answer_accuracy": 0.9344076035409421, + "avg_response_len": 283.59606312243125, + "resp_tokens_sum": 229635673, + "pass_at_k": { + "pass@1": 0.9344076035409421, + "pass@2": 0.9533806316762856, + "pass@4": 0.9661068553298521, + "pass@8": 0.9750141706759139, + "pass@16": 0.9814565557468075, + "pass@32": 0.9862101820285257, + "pass@64": 0.9896272451161039, + "pass@128": 0.9920961112867531 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9496220501474927, + "pass@2": 0.9594972330615753, + "pass@4": 0.966436561856759, + "pass@8": 0.9727340543343289, + "pass@16": 0.9784618657970596, + "pass@32": 0.9825567911391251, + "pass@64": 0.985620534223565, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996177370030581, + "pass@2": 0.9999838215463894, + "pass@4": 0.9999999635940003, + "pass@8": 0.9999999999999807, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9942460317460318, + "pass@2": 0.9975178883889514, + "pass@4": 0.9992110941489457, + "pass@8": 0.9998670726976857, + "pass@16": 0.9999959700663595, + "pass@32": 0.9999999986446861, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993528106508875, + "pass@2": 0.9999286562922239, + "pass@4": 0.9999986563255037, + "pass@8": 0.9999999997393219, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9939903846153846, + "pass@2": 0.997067355012114, + "pass@4": 0.9988529817186313, + "pass@8": 0.9997549809210944, + "pass@16": 0.9999868735089701, + "pass@32": 0.9999999812154626, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9912852112676056, + "pass@2": 0.9926787595652655, + "pass@4": 0.9935059028712959, + "pass@8": 0.9944525916293162, + "pass@16": 0.9953237078346044, + "pass@32": 0.9963063942759401, + "pass@64": 0.9978928690234709, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9613358711048159, + "pass@2": 0.9751445010149226, + "pass@4": 0.9847932027264298, + "pass@8": 0.9914273628571794, + "pass@16": 0.9961681616814383, + "pass@32": 0.99904074927096, + "pass@64": 0.9999499987724876, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9385151836158192, + "pass@2": 0.9597627924952182, + "pass@4": 0.9762562224743094, + "pass@8": 0.9865212132430582, + "pass@16": 0.9922087450214933, + "pass@32": 0.9959870309300201, + "pass@64": 0.9983853008294012, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9068867554858934, + "pass@2": 0.9370576265643127, + "pass@4": 0.9592523463955722, + "pass@8": 0.9740163899882339, + "pass@16": 0.9836944065507172, + "pass@32": 0.9896518391446572, + "pass@64": 0.9926987166516155, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.9166179906542056, + "pass@2": 0.941457704761204, + "pass@4": 0.9603239408158093, + "pass@8": 0.9735008083823662, + "pass@16": 0.9829294702078442, + "pass@32": 0.9901209671439047, + "pass@64": 0.9952071961226423, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9691112716763006, + "pass@2": 0.9872729962222933, + "pass@4": 0.9933811493397139, + "pass@8": 0.9952925662134263, + "pass@16": 0.9963578351065164, + "pass@32": 0.9974679316498488, + "pass@64": 0.9985359192993869, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8643397177419355, + "pass@2": 0.9029233870967743, + "pass@4": 0.9338913727517938, + "pass@8": 0.9545205921905574, + "pass@16": 0.9671956269598453, + "pass@32": 0.9761510160969632, + "pass@64": 0.9823101727503963, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8985503973988439, + "pass@2": 0.9290570524782674, + "pass@4": 0.9477699960554062, + "pass@8": 0.9613079311183532, + "pass@16": 0.9720480116375539, + "pass@32": 0.9786693409502875, + "pass@64": 0.9811048717975289, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8410714285714286, + "pass@2": 0.8791718222722164, + "pass@4": 0.9039378402699658, + "pass@8": 0.9225013658392592, + "pass@16": 0.9380212263183387, + "pass@32": 0.9522564152117664, + "pass@64": 0.9651583391167925, + "pass@128": 0.9771428571428571 + }, + "19": { + "pass@1": 0.7519707207207207, + "pass@2": 0.8127601026223072, + "pass@4": 0.8581806291668495, + "pass@8": 0.8929533160295472, + "pass@16": 0.9184558230351982, + "pass@32": 0.9365729445914778, + "pass@64": 0.9498045174907821, + "pass@128": 0.960960960960961 + }, + "20": { + "pass@1": 0.7815080705705706, + "pass@2": 0.8459900599418323, + "pass@4": 0.8831622091720508, + "pass@8": 0.9091561650991454, + "pass@16": 0.9291310201543674, + "pass@32": 0.9450531167298203, + "pass@64": 0.9576555137668181, + "pass@128": 0.9669669669669669 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 770509, + "answer_accuracy": 0.9383634547934528, + "avg_response_len": 292.753721745908, + "resp_tokens_sum": 240385936, + "pass_at_k": { + "pass@1": 0.9383634547934528, + "pass@2": 0.9567760147844829, + "pass@4": 0.9693254594052584, + "pass@8": 0.9780153737379047, + "pass@16": 0.9838214676849862, + "pass@32": 0.9877374039694216, + "pass@64": 0.9905971086523647, + "pass@128": 0.9929851909586905 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9213238113496932, + "pass@2": 0.9363610151683497, + "pass@4": 0.9515447243749438, + "pass@8": 0.9635215251668017, + "pass@16": 0.9717501171588053, + "pass@32": 0.9789974182409743, + "pass@64": 0.9873278512974969, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99765625, + "pass@2": 0.9993058041040325, + "pass@4": 0.9999122283010079, + "pass@8": 0.9999981196920988, + "pass@16": 0.9999999995180833, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9952445652173914, + "pass@2": 0.9969466792194455, + "pass@4": 0.997354996114616, + "pass@8": 0.9976154255526752, + "pass@16": 0.9980647799459662, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998656876790831, + "pass@2": 0.9999964747422333, + "pass@4": 0.9999999986570447, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9968388728323699, + "pass@2": 0.9980816331755495, + "pass@4": 0.998801978690525, + "pass@8": 0.9995193591669413, + "pass@16": 0.9999304712731262, + "pass@32": 0.9999991345669063, + "pass@64": 0.999999999995383, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913072183098591, + "pass@2": 0.99444105578352, + "pass@4": 0.9964716870778477, + "pass@8": 0.9975479530508959, + "pass@16": 0.9981170196665599, + "pass@32": 0.9988210098675391, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941757110778443, + "pass@2": 0.9960987228299307, + "pass@4": 0.9969899691617888, + "pass@8": 0.9975118492787143, + "pass@16": 0.9980009131217787, + "pass@32": 0.998746881938581, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9735504518072289, + "pass@2": 0.9807889283512, + "pass@4": 0.9859694000975779, + "pass@8": 0.9903034578261026, + "pass@16": 0.9939776137045641, + "pass@32": 0.9972317237104557, + "pass@64": 0.9994343404291871, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.943255289968652, + "pass@2": 0.9643270370004688, + "pass@4": 0.9795041327122507, + "pass@8": 0.9901062501212899, + "pass@16": 0.9963812763120088, + "pass@32": 0.999159903580148, + "pass@64": 0.9999519319520963, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9459252450980392, + "pass@2": 0.969042242881405, + "pass@4": 0.9843180109314063, + "pass@8": 0.9930887640011501, + "pass@16": 0.9972316564809265, + "pass@32": 0.9991371398635513, + "pass@64": 0.9999152127956612, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9421829312865497, + "pass@2": 0.9626221681171436, + "pass@4": 0.9766211556340545, + "pass@8": 0.9860053568807828, + "pass@16": 0.9912234750641322, + "pass@32": 0.9933893500190114, + "pass@64": 0.994067684369298, + "pass@128": 0.9941520467836257 + }, + "15": { + "pass@1": 0.9615056818181819, + "pass@2": 0.9816444464328327, + "pass@4": 0.9924044471713761, + "pass@8": 0.9970935786701942, + "pass@16": 0.9988697871371285, + "pass@32": 0.9997170217125875, + "pass@64": 0.9999906185608666, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.9047364672364673, + "pass@2": 0.9336404564012831, + "pass@4": 0.9554778179436971, + "pass@8": 0.9713046676762731, + "pass@16": 0.981129930647535, + "pass@32": 0.9863700316854713, + "pass@64": 0.9898460871084831, + "pass@128": 0.9943019943019943 + }, + "17": { + "pass@1": 0.8905764751552795, + "pass@2": 0.9193115891084263, + "pass@4": 0.9371282252373729, + "pass@8": 0.9495247038246433, + "pass@16": 0.9600510648716486, + "pass@32": 0.9696963337203404, + "pass@64": 0.9783483636870083, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8783318014705882, + "pass@2": 0.9212120773506257, + "pass@4": 0.9446521532234942, + "pass@8": 0.9603982535156668, + "pass@16": 0.971534461743965, + "pass@32": 0.9800680670995429, + "pass@64": 0.986866489127435, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7672526041666666, + "pass@2": 0.8213970812242222, + "pass@4": 0.8611331446292425, + "pass@8": 0.8925303692099814, + "pass@16": 0.9155570589381155, + "pass@32": 0.9306472438901101, + "pass@64": 0.9397489108469029, + "pass@128": 0.9464285714285714 + }, + "20": { + "pass@1": 0.7179079341317365, + "pass@2": 0.797639710736008, + "pass@4": 0.8541305002480968, + "pass@8": 0.8923328681460105, + "pass@16": 0.9176848689576249, + "pass@32": 0.9338587403043053, + "pass@64": 0.9451756698292793, + "pass@128": 0.9550898203592815 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 747861, + "answer_accuracy": 0.9334820358683495, + "avg_response_len": 313.2888777660169, + "resp_tokens_sum": 250992011, + "pass_at_k": { + "pass@1": 0.9334820358683495, + "pass@2": 0.9537701733126235, + "pass@4": 0.9670815015054736, + "pass@8": 0.9760319147552091, + "pass@16": 0.9821892780315375, + "pass@32": 0.9865542675416312, + "pass@64": 0.9896955198919034, + "pass@128": 0.9918517335037546 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9504897388059701, + "pass@2": 0.9637266423786578, + "pass@4": 0.9740664414149726, + "pass@8": 0.9809101215824065, + "pass@16": 0.9856060890227171, + "pass@32": 0.9899456026975152, + "pass@64": 0.9932744178001089, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996573464912281, + "pass@2": 0.9999622271722615, + "pass@4": 0.9999996258691347, + "pass@8": 0.9999999999868394, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9943244485294118, + "pass@2": 0.9970045449282076, + "pass@4": 0.9982432999183926, + "pass@8": 0.9991491885263906, + "pass@16": 0.999780019461051, + "pass@32": 0.9999893918747133, + "pass@64": 0.9999999972066469, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9955992879746836, + "pass@2": 0.996529421658527, + "pass@4": 0.9969824845865787, + "pass@8": 0.9972195335748744, + "pass@16": 0.9975798614235771, + "pass@32": 0.9982246087909897, + "pass@64": 0.9992150902023323, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9915540540540541, + "pass@2": 0.9943531129554752, + "pass@4": 0.9959725785965943, + "pass@8": 0.9974309201820005, + "pass@16": 0.9988093286723544, + "pass@32": 0.9997137994250378, + "pass@64": 0.9999907019175951, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928155144694534, + "pass@2": 0.9952599248044157, + "pass@4": 0.9962277089760885, + "pass@8": 0.9966958564196817, + "pass@16": 0.9967827847936206, + "pass@32": 0.9967845657060646, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9673611111111111, + "pass@2": 0.9821323115860517, + "pass@4": 0.9926942709542262, + "pass@8": 0.9980986789972592, + "pass@16": 0.9997494773091639, + "pass@32": 0.9999937908674831, + "pass@64": 0.9999999994787548, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.950759747706422, + "pass@2": 0.9713404337932532, + "pass@4": 0.9855429708327742, + "pass@8": 0.9928148852591774, + "pass@16": 0.995613443895582, + "pass@32": 0.9967029940509836, + "pass@64": 0.9969374409579194, + "pass@128": 0.9969418960244648 + }, + "13": { + "pass@1": 0.8924334490740741, + "pass@2": 0.9274767303392636, + "pass@4": 0.9551435529470846, + "pass@8": 0.9738012340666778, + "pass@16": 0.9852240935964495, + "pass@32": 0.9924642039682082, + "pass@64": 0.9960781632221886, + "pass@128": 0.9969135802469136 + }, + "14": { + "pass@1": 0.9461007047477745, + "pass@2": 0.9686999842286034, + "pass@4": 0.9829498372236113, + "pass@8": 0.9910325386471672, + "pass@16": 0.995148432923748, + "pass@32": 0.9973034567467718, + "pass@64": 0.9984965964597918, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9357397762345679, + "pass@2": 0.9589940580344121, + "pass@4": 0.9725759832682954, + "pass@8": 0.981825320220379, + "pass@16": 0.987668589719086, + "pass@32": 0.9901856943660934, + "pass@64": 0.9907199216941467, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.9002581120943953, + "pass@2": 0.9337901830302185, + "pass@4": 0.9539900026324142, + "pass@8": 0.9649273623901918, + "pass@16": 0.971576240013085, + "pass@32": 0.9763674951513314, + "pass@64": 0.9798234041567667, + "pass@128": 0.9823008849557522 + }, + "17": { + "pass@1": 0.8558923192771084, + "pass@2": 0.8873935703443695, + "pass@4": 0.9093786171118668, + "pass@8": 0.9264764595802648, + "pass@16": 0.9403468570920928, + "pass@32": 0.9517275677742213, + "pass@64": 0.963140868562918, + "pass@128": 0.9759036144578314 + }, + "18": { + "pass@1": 0.8797631048387097, + "pass@2": 0.9189547879095755, + "pass@4": 0.9408445722510488, + "pass@8": 0.9570493000821247, + "pass@16": 0.9697178547928714, + "pass@32": 0.9798583276175898, + "pass@64": 0.9874463456793682, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7480173716012085, + "pass@2": 0.8153925410947498, + "pass@4": 0.8618116894187314, + "pass@8": 0.8955726691120511, + "pass@16": 0.921278465008222, + "pass@32": 0.9415106476448225, + "pass@64": 0.9571645244414844, + "pass@128": 0.9667673716012085 + }, + "20": { + "pass@1": 0.7336007882882883, + "pass@2": 0.8098284061620676, + "pass@4": 0.8578734074344312, + "pass@8": 0.8917625881142276, + "pass@16": 0.9171084516911536, + "pass@32": 0.9343042775669052, + "pass@64": 0.9457695347626426, + "pass@128": 0.954954954954955 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_id_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2d10680a08757bd558353c51c4d807282a3efe73 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b70b322a60c3efe38633643246d20a17bfa00687d8a3b6ccc312ef6ffc7d0adc +size 1790074537 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_metrics.json b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..550c6dc1f61fc9848f340b4955051e32f00dfcde --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-39470", + "id": { + "answer_accuracy": 0.9900894097222223, + "avg_response_len": 200.35493229166667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9900894097222223, + "pass@2": 0.9933217410323687, + "pass@4": 0.995405526152981, + "pass@8": 0.9969335693559814, + "pass@16": 0.9980247772956123, + "pass@32": 0.9987844085168759, + "pass@64": 0.9993596451557488, + "pass@128": 0.9997777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.93928125, + "pass@2": 0.9548156988188969, + "pass@4": 0.9676834842519687, + "pass@8": 0.9778111478085322, + "pass@16": 0.985276794281646, + "pass@32": 0.9905220424657489, + "pass@64": 0.9947966796012669, + "pass@128": 0.998 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997375, + "pass@2": 0.9990730807086614, + "pass@4": 0.99972067904012, + "pass@8": 0.9999495167856408, + "pass@16": 0.9999981086322938, + "pass@32": 0.999999998952485, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99903125, + "pass@2": 0.9996382874015749, + "pass@4": 0.9999106776340456, + "pass@8": 0.9999933896260147, + "pass@16": 0.9999999734303598, + "pass@32": 0.9999999999999412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996171875, + "pass@2": 0.998694758858268, + "pass@4": 0.9992537027559055, + "pass@8": 0.9995214498236242, + "pass@16": 0.9997843450038431, + "pass@32": 0.9999637597411362, + "pass@64": 0.9999996944261581, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999640625, + "pass@2": 0.9999843750000001, + "pass@4": 0.999999904199475, + "pass@8": 0.9999999999978996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9954140625, + "pass@2": 0.9969405757874017, + "pass@4": 0.9980064220097488, + "pass@8": 0.998911192299082, + "pass@16": 0.9995661235648396, + "pass@32": 0.9999041746513426, + "pass@64": 0.9999969039049552, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9921484375, + "pass@2": 0.9959159694881891, + "pass@4": 0.9979444792838394, + "pass@8": 0.9992674716908343, + "pass@16": 0.999862717226493, + "pass@32": 0.9999925922934341, + "pass@64": 0.9999999947647286, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.99175, + "pass@2": 0.9948329232283467, + "pass@4": 0.9961303862017246, + "pass@8": 0.9969479561722048, + "pass@16": 0.997734933521036, + "pass@32": 0.9986771085477952, + "pass@64": 0.9994435337046285, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.93928125, + "2": 0.9999921875, + "3": 0.997375, + "4": 0.99903125, + "5": 0.996171875, + "6": 0.999640625, + "7": 0.9954140625, + "8": 0.9921484375, + "9": 0.99175 + }, + "per_op_avg_response_len": { + "10": 281.686140625, + "2": 115.6301328125, + "3": 145.0052890625, + "4": 159.9863984375, + "5": 173.589984375, + "6": 199.2454453125, + "7": 224.2465390625, + "8": 243.9378125, + "9": 259.8666484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.686140625, + "2": 115.6301328125, + "3": 145.0052890625, + "4": 159.9863984375, + "5": 173.589984375, + "6": 199.2454453125, + "7": 224.2465390625, + "8": 243.9378125, + "9": 259.8666484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375117, + "answer_accuracy": 0.9897337259371833, + "avg_response_len": 190.78254021023304, + "resp_tokens_sum": 72308109, + "pass_at_k": { + "pass@1": 0.9897337259371833, + "pass@2": 0.9931395294338211, + "pass@4": 0.9951424401800337, + "pass@8": 0.9966416403004525, + "pass@16": 0.9978827355359013, + "pass@32": 0.998927606005656, + "pass@64": 0.9997112386232214, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392514749262537, + "pass@2": 0.9528702268134623, + "pass@4": 0.9647196697204883, + "pass@8": 0.9752039016665607, + "pass@16": 0.9841804774412467, + "pass@32": 0.9916424699073122, + "pass@64": 0.9975593656716509, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9964162844036697, + "pass@2": 0.9992663259409089, + "pass@4": 0.9999513056854131, + "pass@8": 0.9999997214289341, + "pass@16": 0.9999999999945326, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99814453125, + "pass@2": 0.9990688053641733, + "pass@4": 0.9997314116985375, + "pass@8": 0.9999793655718723, + "pass@16": 0.9999999169698948, + "pass@32": 0.9999999999998159, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982886904761905, + "pass@2": 0.9999117297837768, + "pass@4": 0.9999994979794193, + "pass@8": 0.9999999999856092, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9999620591494178, + "pass@4": 0.9999997176625378, + "pass@8": 0.9999999999937114, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9974112426035503, + "pass@2": 0.9990299439733494, + "pass@4": 0.999743682649876, + "pass@8": 0.9999804637850213, + "pass@16": 0.9999999213916163, + "pass@32": 0.9999999999998258, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9911608573717948, + "pass@2": 0.9963473052947709, + "pass@4": 0.9984855673088942, + "pass@8": 0.9994635469329705, + "pass@16": 0.9999228935849399, + "pass@32": 0.9999990402568896, + "pass@64": 0.9999999999948799, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9897447183098591, + "pass@2": 0.9935465648220028, + "pass@4": 0.9950367868629096, + "pass@8": 0.9961759013482411, + "pass@16": 0.9975146715824619, + "pass@32": 0.9990370240116533, + "pass@64": 0.9999221200063837, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386410, + "answer_accuracy": 0.9891311025557011, + "avg_response_len": 198.29557718299475, + "resp_tokens_sum": 77465357, + "pass_at_k": { + "pass@1": 0.9891311025557011, + "pass@2": 0.992221563438458, + "pass@4": 0.9943991502618527, + "pass@8": 0.9960482286051604, + "pass@16": 0.9972822537933906, + "pass@32": 0.9982065468352347, + "pass@64": 0.9990566747280499, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9263564033742331, + "pass@2": 0.943031511158881, + "pass@4": 0.9574623036959338, + "pass@8": 0.9692292659689711, + "pass@16": 0.9785950061939993, + "pass@32": 0.985816650956142, + "pass@64": 0.9927118784293236, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9957859848484848, + "pass@2": 0.9979222590073968, + "pass@4": 0.9992018259081249, + "pass@8": 0.9998472966011496, + "pass@16": 0.999994268588126, + "pass@32": 0.9999999968257123, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996764053254438, + "pass@2": 0.9999919920328006, + "pass@4": 0.999999990293373, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996286231884058, + "pass@2": 0.9990335786830994, + "pass@4": 0.9998567722512948, + "pass@8": 0.9999954258079576, + "pass@16": 0.9999999965927238, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997089899713467, + "pass@2": 0.9999936545360197, + "pass@4": 0.9999999959711339, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9963872832369942, + "pass@2": 0.9973061080515223, + "pass@4": 0.9977670850211643, + "pass@8": 0.9983013813782712, + "pass@16": 0.999038785005553, + "pass@32": 0.9997319443095561, + "pass@64": 0.9999910524153849, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9921654929577465, + "pass@2": 0.9949418459576357, + "pass@4": 0.9969772593566649, + "pass@8": 0.998742979098145, + "pass@16": 0.9996984788969664, + "pass@32": 0.9999800029461254, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932634730538922, + "pass@2": 0.9957594653213259, + "pass@4": 0.9968023138200539, + "pass@8": 0.9971748134078343, + "pass@16": 0.9973801780816433, + "pass@32": 0.9977544910178684, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379056, + "answer_accuracy": 0.9914211583528624, + "avg_response_len": 211.9481712420489, + "resp_tokens_sum": 81035416, + "pass_at_k": { + "pass@1": 0.9914211583528624, + "pass@2": 0.9946264851100179, + "pass@4": 0.9966945977250001, + "pass@8": 0.9981275639676072, + "pass@16": 0.9989242641986213, + "pass@32": 0.9992328939832608, + "pass@64": 0.9993206754497388, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9518889925373134, + "pass@2": 0.9682519978845928, + "pass@4": 0.9806292692891003, + "pass@8": 0.9888008490677124, + "pass@16": 0.9928884788352844, + "pass@32": 0.9939672147924427, + "pass@64": 0.9940297978227394, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999960543835081, + "pass@4": 0.9999999986335526, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992233187134503, + "pass@2": 0.9998215683565871, + "pass@4": 0.9999901437758876, + "pass@8": 0.9999999784883499, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940946691176471, + "pass@2": 0.9972234686197314, + "pass@4": 0.9979508072520346, + "pass@8": 0.9985971409541529, + "pass@16": 0.9993657240569218, + "pass@32": 0.999893411003342, + "pass@64": 0.9999991012534062, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998767744479495, + "pass@2": 0.9999976713281503, + "pass@4": 0.9999999997042957, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9922122231012658, + "pass@2": 0.9943055105651352, + "pass@4": 0.9964102716986326, + "pass@8": 0.9984351822242498, + "pass@16": 0.9996795269701011, + "pass@32": 0.9999902592414398, + "pass@64": 0.9999999992697213, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930555555555556, + "pass@2": 0.9965503150788586, + "pass@4": 0.9984686342698152, + "pass@8": 0.9996429053090278, + "pass@16": 0.9999814246833897, + "pass@32": 0.999999972034264, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9924135852090032, + "pass@2": 0.9953062099653139, + "pass@4": 0.9966570869758643, + "pass@8": 0.9975856061587218, + "pass@16": 0.9983673492925828, + "pass@32": 0.9992571222626693, + "pass@64": 0.9999073347342836, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.87708359375, + "avg_response_len": 383.22090859375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.87708359375, + "pass@2": 0.91426534202754, + "pass@4": 0.9388444801649733, + "pass@8": 0.9554630114306502, + "pass@16": 0.9668432607513274, + "pass@32": 0.9750626066830802, + "pass@64": 0.981182998422084, + "pass@128": 0.9854 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9618828125, + "pass@2": 0.9761438238188973, + "pass@4": 0.986121069178853, + "pass@8": 0.9922099412986207, + "pass@16": 0.9960378852478021, + "pass@32": 0.9983668059219715, + "pass@64": 0.9996202756361483, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9383125, + "pass@2": 0.9615771407480318, + "pass@4": 0.978793215129359, + "pass@8": 0.9894155873886409, + "pass@16": 0.9950864203105703, + "pass@32": 0.997908781909562, + "pass@64": 0.9993572832556711, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9205, + "pass@2": 0.9507199803149609, + "pass@4": 0.9714700293400825, + "pass@8": 0.984238983951573, + "pass@16": 0.9912929665015434, + "pass@32": 0.9952209725742949, + "pass@64": 0.9975568096855626, + "pass@128": 0.999 + }, + "14": { + "pass@1": 0.93465625, + "pass@2": 0.9593348917322839, + "pass@4": 0.9754503624859403, + "pass@8": 0.9856374604637109, + "pass@16": 0.9916007829974561, + "pass@32": 0.9954488510683017, + "pass@64": 0.9977443220289764, + "pass@128": 0.999 + }, + "15": { + "pass@1": 0.9428515625, + "pass@2": 0.9662106299212605, + "pass@4": 0.9796231772590922, + "pass@8": 0.9872753071158634, + "pass@16": 0.9909704662467119, + "pass@32": 0.9928044636946507, + "pass@64": 0.9942301201324907, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.88575, + "pass@2": 0.921519315944882, + "pass@4": 0.9451636838207728, + "pass@8": 0.9600559276369797, + "pass@16": 0.9696556244998803, + "pass@32": 0.9768272050369899, + "pass@64": 0.9826039588965715, + "pass@128": 0.987 + }, + "17": { + "pass@1": 0.8736875, + "pass@2": 0.908056717519686, + "pass@4": 0.92949713648294, + "pass@8": 0.945244242597247, + "pass@16": 0.9580096201080402, + "pass@32": 0.9686979784203856, + "pass@64": 0.9772030976782187, + "pass@128": 0.983 + }, + "18": { + "pass@1": 0.8517265625, + "pass@2": 0.8972039862204734, + "pass@4": 0.9240943790776152, + "pass@8": 0.9425342533086204, + "pass@16": 0.9561428287515371, + "pass@32": 0.9664975999366425, + "pass@64": 0.9743487856242005, + "pass@128": 0.98 + }, + "19": { + "pass@1": 0.7336875, + "pass@2": 0.800419168307087, + "pass@4": 0.8467187432508434, + "pass@8": 0.8807538663080849, + "pass@16": 0.9067730207449964, + "pass@32": 0.9271565813749784, + "pass@64": 0.943155816270281, + "pass@128": 0.954 + }, + "20": { + "pass@1": 0.72778125, + "pass@2": 0.8014677657480309, + "pass@4": 0.8515130056242972, + "pass@8": 0.887264544237168, + "pass@16": 0.9128629921046956, + "pass@32": 0.9316968268930037, + "pass@64": 0.9460095150126977, + "pass@128": 0.956 + } + }, + "per_op_accuracy": { + "11": 0.9618828125, + "12": 0.9383125, + "13": 0.9205, + "14": 0.93465625, + "15": 0.9428515625, + "16": 0.88575, + "17": 0.8736875, + "18": 0.8517265625, + "19": 0.7336875, + "20": 0.72778125 + }, + "per_op_avg_response_len": { + "11": 301.180265625, + "12": 313.2810078125, + "13": 334.311234375, + "14": 361.1121640625, + "15": 381.6251484375, + "16": 393.495109375, + "17": 414.98365625, + "18": 433.115546875, + "19": 437.9015859375, + "20": 461.2033671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 301.180265625, + "12": 313.2810078125, + "13": 334.311234375, + "14": 361.1121640625, + "15": 381.6251484375, + "16": 393.495109375, + "17": 414.98365625, + "18": 433.115546875, + "19": 437.9015859375, + "20": 461.2033671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 376466, + "answer_accuracy": 0.8740388187221396, + "avg_response_len": 365.3849623885587, + "resp_tokens_sum": 157378611, + "pass_at_k": { + "pass@1": 0.8740388187221396, + "pass@2": 0.911976832785394, + "pass@4": 0.9367364605984453, + "pass@8": 0.9535655276162108, + "pass@16": 0.9653974041484722, + "pass@32": 0.9740992796976093, + "pass@64": 0.9805766825281853, + "pass@128": 0.9851411589895989 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9560685198300283, + "pass@2": 0.972025147219558, + "pass@4": 0.9833676431380923, + "pass@8": 0.9902832202553371, + "pass@16": 0.9946957855807873, + "pass@32": 0.997549645536328, + "pass@64": 0.9992147611382037, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.943215925141243, + "pass@2": 0.9660565138573776, + "pass@4": 0.9814934266691235, + "pass@8": 0.9904757233909742, + "pass@16": 0.9957168814771249, + "pass@32": 0.9984134645336938, + "pass@64": 0.9996455904087194, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9108787225705329, + "pass@2": 0.9434656776837063, + "pass@4": 0.9667659061230197, + "pass@8": 0.9808162550485344, + "pass@16": 0.9881944688814267, + "pass@32": 0.9930606223263609, + "pass@64": 0.9966675055170503, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9186380451713395, + "pass@2": 0.9460888249074009, + "pass@4": 0.9653372817299704, + "pass@8": 0.9779994019698924, + "pass@16": 0.9860626693919856, + "pass@32": 0.9919049064530143, + "pass@64": 0.9963014598578678, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9521540823699421, + "pass@2": 0.9741484513677119, + "pass@4": 0.985112245633313, + "pass@8": 0.9908469302823427, + "pass@16": 0.9934400862091718, + "pass@32": 0.994992084796452, + "pass@64": 0.9963530196364373, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8628276209677419, + "pass@2": 0.9018546005842007, + "pass@4": 0.9287450191709906, + "pass@8": 0.9468683936291992, + "pass@16": 0.9613419346632679, + "pass@32": 0.9736111775442221, + "pass@64": 0.9823099408096163, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.884709176300578, + "pass@2": 0.9211279214646582, + "pass@4": 0.9415992674867953, + "pass@8": 0.9565759689885837, + "pass@16": 0.9692118557186207, + "pass@32": 0.9795491000031725, + "pass@64": 0.98639472632689, + "pass@128": 0.9884393063583815 + }, + "18": { + "pass@1": 0.82296875, + "pass@2": 0.8701905230596182, + "pass@4": 0.8998161420001068, + "pass@8": 0.9205166862174053, + "pass@16": 0.9355630508173823, + "pass@32": 0.9477589149914283, + "pass@64": 0.9590373492455058, + "pass@128": 0.9685714285714285 + }, + "19": { + "pass@1": 0.7173188813813813, + "pass@2": 0.7844284983802695, + "pass@4": 0.8315280263390493, + "pass@8": 0.8687886876667537, + "pass@16": 0.8990525439832475, + "pass@32": 0.9216565068026786, + "pass@64": 0.9364336919565242, + "pass@128": 0.9459459459459459 + }, + "20": { + "pass@1": 0.7638419669669669, + "pass@2": 0.8345680818613895, + "pass@4": 0.8795241003095331, + "pass@8": 0.9096289064963395, + "pass@16": 0.9287254882088332, + "pass@32": 0.9412760341681837, + "pass@64": 0.9527062784829503, + "pass@128": 0.963963963963964 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 380494, + "answer_accuracy": 0.8839159604519774, + "avg_response_len": 378.29514895554564, + "resp_tokens_sum": 162842443, + "pass_at_k": { + "pass@1": 0.8839159604519774, + "pass@2": 0.9195300116365964, + "pass@4": 0.9433888869656264, + "pass@8": 0.9595429045096775, + "pass@16": 0.9702692032553104, + "pass@32": 0.9776433391303953, + "pass@64": 0.9829795626977784, + "pass@128": 0.9866190900981266 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9698795180722891, + "pass@2": 0.9801018789725835, + "pass@4": 0.9859459858556837, + "pass@8": 0.9903607486554723, + "pass@16": 0.9944354686155655, + "pass@32": 0.997734424893915, + "pass@64": 0.9996912428943647, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9312793887147336, + "pass@2": 0.9578514921136424, + "pass@4": 0.9785946927088653, + "pass@8": 0.9911220322102386, + "pass@16": 0.9969256254585147, + "pass@32": 0.999356859530079, + "pass@64": 0.9999770727927124, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9410014005602241, + "pass@2": 0.9629413281060455, + "pass@4": 0.9783460898795218, + "pass@8": 0.9885164313887403, + "pass@16": 0.9938448297567265, + "pass@32": 0.99557976291731, + "pass@64": 0.9965040977444787, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.938093932748538, + "pass@2": 0.9617224553345302, + "pass@4": 0.9780836096145873, + "pass@8": 0.9891690673671412, + "pass@16": 0.9953444109429628, + "pass@32": 0.9985866789814201, + "pass@64": 0.9998999204723729, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.946875, + "pass@2": 0.9708445180147938, + "pass@4": 0.9852852896228875, + "pass@8": 0.9932527012437333, + "pass@16": 0.9965859651004563, + "pass@32": 0.9976875706827749, + "pass@64": 0.9984848257468957, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8979255698005698, + "pass@2": 0.9284205560042177, + "pass@4": 0.9476878840999577, + "pass@8": 0.9601126751922175, + "pass@16": 0.9681501715762721, + "pass@32": 0.9745545081734679, + "pass@64": 0.9806737994273104, + "pass@128": 0.9857549857549858 + }, + "17": { + "pass@1": 0.883103649068323, + "pass@2": 0.9193444484520957, + "pass@4": 0.9431839631265031, + "pass@8": 0.9588158770909269, + "pass@16": 0.969734921800441, + "pass@32": 0.9777583996276256, + "pass@64": 0.9834804448754739, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8681985294117647, + "pass@2": 0.9134885942566008, + "pass@4": 0.9388149804803808, + "pass@8": 0.956182315324883, + "pass@16": 0.969199828469146, + "pass@32": 0.9786850953286746, + "pass@64": 0.98495366662752, + "pass@128": 0.9882352941176471 + }, + "19": { + "pass@1": 0.7562779017857143, + "pass@2": 0.8173970929415075, + "pass@4": 0.8586776217704924, + "pass@8": 0.8873359588818277, + "pass@16": 0.9093487259281797, + "pass@32": 0.9286139494295144, + "pass@64": 0.9465647885804772, + "pass@128": 0.9613095238095238 + }, + "20": { + "pass@1": 0.7049962574850299, + "pass@2": 0.7821469511763871, + "pass@4": 0.8387314392587147, + "pass@8": 0.8803005442248173, + "pass@16": 0.9090276962957424, + "pass@32": 0.927911101416634, + "pass@64": 0.9395962270441276, + "pass@128": 0.9461077844311377 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 365707, + "answer_accuracy": 0.8731925236858191, + "avg_response_len": 406.62655915724326, + "resp_tokens_sum": 170301709, + "pass_at_k": { + "pass@1": 0.8731925236858191, + "pass@2": 0.9112078083187384, + "pass@4": 0.9363416212929875, + "pass@8": 0.9532210654070638, + "pass@16": 0.9648089889994019, + "pass@32": 0.9734008071982895, + "pass@64": 0.9799600177753136, + "pass@128": 0.9844132029339854 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9599702380952381, + "pass@2": 0.9765876921634795, + "pass@4": 0.989391186815934, + "pass@8": 0.9963180825233957, + "pass@16": 0.9992307852616401, + "pass@32": 0.9999490539138667, + "pass@64": 0.9999999121061375, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.939865252293578, + "pass@2": 0.9603624124106047, + "pass@4": 0.9760637159460566, + "pass@8": 0.9866032202849236, + "pass@16": 0.9926096934140741, + "pass@32": 0.9959497775979793, + "pass@64": 0.9984405444345842, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9073832947530864, + "pass@2": 0.9443962191358023, + "pass@4": 0.9685251580242282, + "pass@8": 0.9828957795842923, + "pass@16": 0.9915318725469653, + "pass@32": 0.996952650279957, + "pass@64": 0.9995923226260632, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9464252596439169, + "pass@2": 0.9695290777821917, + "pass@4": 0.9824109808974172, + "pass@8": 0.9893288765335709, + "pass@16": 0.9930767880716179, + "pass@32": 0.9956401539621311, + "pass@64": 0.996931099741985, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9288194444444444, + "pass@2": 0.9530141197628071, + "pass@4": 0.9679944280691763, + "pass@8": 0.9773730797152483, + "pass@16": 0.9826136664666306, + "pass@32": 0.9854947655239579, + "pass@64": 0.9876295766722463, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.8941049041297935, + "pass@2": 0.932356267275219, + "pass@4": 0.9575642494356341, + "pass@8": 0.9720565681989967, + "pass@16": 0.9788168570235897, + "pass@32": 0.9821212614436396, + "pass@64": 0.9848713086920479, + "pass@128": 0.9882005899705014 + }, + "17": { + "pass@1": 0.8530685240963856, + "pass@2": 0.8834865791907793, + "pass@4": 0.9036101018246212, + "pass@8": 0.9202718219997545, + "pass@16": 0.9349628711134209, + "pass@32": 0.9486017624674481, + "pass@64": 0.9615355997566628, + "pass@128": 0.9728915662650602 + }, + "18": { + "pass@1": 0.8661290322580645, + "pass@2": 0.9098425196850394, + "pass@4": 0.9353601161749935, + "pass@8": 0.9524239545873169, + "pass@16": 0.9650574815675628, + "pass@32": 0.9742872492835252, + "pass@64": 0.9800047314029565, + "pass@128": 0.9838709677419355 + }, + "19": { + "pass@1": 0.7272233761329305, + "pass@2": 0.7992720698432335, + "pass@4": 0.8498617207403449, + "pass@8": 0.886109822147316, + "pass@16": 0.9119255337966973, + "pass@32": 0.9312105063485466, + "pass@64": 0.946458090289178, + "pass@128": 0.9546827794561934 + }, + "20": { + "pass@1": 0.714573948948949, + "pass@2": 0.7877462846704968, + "pass@4": 0.8363218603868204, + "pass@8": 0.8718850949032955, + "pass@16": 0.9008473092143473, + "pass@32": 0.9259147136391661, + "pass@64": 0.945745298634043, + "pass@128": 0.9579579579579579 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9306126644736842, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9306126644736842, + "pass@2": 0.9517131099772012, + "pass@4": 0.9656365545803396, + "pass@8": 0.9751069599215995, + "pass@16": 0.9816134527986127, + "pass@32": 0.9862992496569865, + "pass@64": 0.9897929889801356, + "pass@128": 0.9922105263157894 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.93928125, + "pass@2": 0.9548156988188969, + "pass@4": 0.9676834842519687, + "pass@8": 0.9778111478085322, + "pass@16": 0.985276794281646, + "pass@32": 0.9905220424657489, + "pass@64": 0.9947966796012669, + "pass@128": 0.998 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997375, + "pass@2": 0.9990730807086614, + "pass@4": 0.99972067904012, + "pass@8": 0.9999495167856408, + "pass@16": 0.9999981086322938, + "pass@32": 0.999999998952485, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99903125, + "pass@2": 0.9996382874015749, + "pass@4": 0.9999106776340456, + "pass@8": 0.9999933896260147, + "pass@16": 0.9999999734303598, + "pass@32": 0.9999999999999412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996171875, + "pass@2": 0.998694758858268, + "pass@4": 0.9992537027559055, + "pass@8": 0.9995214498236242, + "pass@16": 0.9997843450038431, + "pass@32": 0.9999637597411362, + "pass@64": 0.9999996944261581, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999640625, + "pass@2": 0.9999843750000001, + "pass@4": 0.999999904199475, + "pass@8": 0.9999999999978996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9954140625, + "pass@2": 0.9969405757874017, + "pass@4": 0.9980064220097488, + "pass@8": 0.998911192299082, + "pass@16": 0.9995661235648396, + "pass@32": 0.9999041746513426, + "pass@64": 0.9999969039049552, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9921484375, + "pass@2": 0.9959159694881891, + "pass@4": 0.9979444792838394, + "pass@8": 0.9992674716908343, + "pass@16": 0.999862717226493, + "pass@32": 0.9999925922934341, + "pass@64": 0.9999999947647286, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.99175, + "pass@2": 0.9948329232283467, + "pass@4": 0.9961303862017246, + "pass@8": 0.9969479561722048, + "pass@16": 0.997734933521036, + "pass@32": 0.9986771085477952, + "pass@64": 0.9994435337046285, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9618828125, + "pass@2": 0.9761438238188973, + "pass@4": 0.986121069178853, + "pass@8": 0.9922099412986207, + "pass@16": 0.9960378852478021, + "pass@32": 0.9983668059219715, + "pass@64": 0.9996202756361483, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9383125, + "pass@2": 0.9615771407480318, + "pass@4": 0.978793215129359, + "pass@8": 0.9894155873886409, + "pass@16": 0.9950864203105703, + "pass@32": 0.997908781909562, + "pass@64": 0.9993572832556711, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9205, + "pass@2": 0.9507199803149609, + "pass@4": 0.9714700293400825, + "pass@8": 0.984238983951573, + "pass@16": 0.9912929665015434, + "pass@32": 0.9952209725742949, + "pass@64": 0.9975568096855626, + "pass@128": 0.999 + }, + "14": { + "pass@1": 0.93465625, + "pass@2": 0.9593348917322839, + "pass@4": 0.9754503624859403, + "pass@8": 0.9856374604637109, + "pass@16": 0.9916007829974561, + "pass@32": 0.9954488510683017, + "pass@64": 0.9977443220289764, + "pass@128": 0.999 + }, + "15": { + "pass@1": 0.9428515625, + "pass@2": 0.9662106299212605, + "pass@4": 0.9796231772590922, + "pass@8": 0.9872753071158634, + "pass@16": 0.9909704662467119, + "pass@32": 0.9928044636946507, + "pass@64": 0.9942301201324907, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.88575, + "pass@2": 0.921519315944882, + "pass@4": 0.9451636838207728, + "pass@8": 0.9600559276369797, + "pass@16": 0.9696556244998803, + "pass@32": 0.9768272050369899, + "pass@64": 0.9826039588965715, + "pass@128": 0.987 + }, + "17": { + "pass@1": 0.8736875, + "pass@2": 0.908056717519686, + "pass@4": 0.92949713648294, + "pass@8": 0.945244242597247, + "pass@16": 0.9580096201080402, + "pass@32": 0.9686979784203856, + "pass@64": 0.9772030976782187, + "pass@128": 0.983 + }, + "18": { + "pass@1": 0.8517265625, + "pass@2": 0.8972039862204734, + "pass@4": 0.9240943790776152, + "pass@8": 0.9425342533086204, + "pass@16": 0.9561428287515371, + "pass@32": 0.9664975999366425, + "pass@64": 0.9743487856242005, + "pass@128": 0.98 + }, + "19": { + "pass@1": 0.7336875, + "pass@2": 0.800419168307087, + "pass@4": 0.8467187432508434, + "pass@8": 0.8807538663080849, + "pass@16": 0.9067730207449964, + "pass@32": 0.9271565813749784, + "pass@64": 0.943155816270281, + "pass@128": 0.954 + }, + "20": { + "pass@1": 0.72778125, + "pass@2": 0.8014677657480309, + "pass@4": 0.8515130056242972, + "pass@8": 0.887264544237168, + "pass@16": 0.9128629921046956, + "pass@32": 0.9316968268930037, + "pass@64": 0.9460095150126977, + "pass@128": 0.956 + } + }, + "per_op_accuracy": { + "10": 0.93928125, + "2": 0.9999921875, + "3": 0.997375, + "4": 0.99903125, + "5": 0.996171875, + "6": 0.999640625, + "7": 0.9954140625, + "8": 0.9921484375, + "9": 0.99175, + "11": 0.9618828125, + "12": 0.9383125, + "13": 0.9205, + "14": 0.93465625, + "15": 0.9428515625, + "16": 0.88575, + "17": 0.8736875, + "18": 0.8517265625, + "19": 0.7336875, + "20": 0.72778125 + }, + "per_op_avg_response_len": { + "10": 281.686140625, + "2": 115.6301328125, + "3": 145.0052890625, + "4": 159.9863984375, + "5": 173.589984375, + "6": 199.2454453125, + "7": 224.2465390625, + "8": 243.9378125, + "9": 259.8666484375, + "11": 301.180265625, + "12": 313.2810078125, + "13": 334.311234375, + "14": 361.1121640625, + "15": 381.6251484375, + "16": 393.495109375, + "17": 414.98365625, + "18": 433.115546875, + "19": 437.9015859375, + "20": 461.2033671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.686140625, + "2": 115.6301328125, + "3": 145.0052890625, + "4": 159.9863984375, + "5": 173.589984375, + "6": 199.2454453125, + "7": 224.2465390625, + "8": 243.9378125, + "9": 259.8666484375, + "11": 301.180265625, + "12": 313.2810078125, + "13": 334.311234375, + "14": 361.1121640625, + "15": 381.6251484375, + "16": 393.495109375, + "17": 414.98365625, + "18": 433.115546875, + "19": 437.9015859375, + "20": 461.2033671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 751583, + "answer_accuracy": 0.9281919360575404, + "avg_response_len": 283.65910527979764, + "resp_tokens_sum": 229686720, + "pass_at_k": { + "pass@1": 0.9281919360575404, + "pass@2": 0.9499665173848083, + "pass@4": 0.9640744475635182, + "pass@8": 0.9737280900028716, + "pass@16": 0.980602757648027, + "pass@32": 0.9857206319262101, + "pass@64": 0.9895329614718181, + "pass@128": 0.9920961112867531 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392514749262537, + "pass@2": 0.9528702268134623, + "pass@4": 0.9647196697204883, + "pass@8": 0.9752039016665607, + "pass@16": 0.9841804774412467, + "pass@32": 0.9916424699073122, + "pass@64": 0.9975593656716509, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9964162844036697, + "pass@2": 0.9992663259409089, + "pass@4": 0.9999513056854131, + "pass@8": 0.9999997214289341, + "pass@16": 0.9999999999945326, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99814453125, + "pass@2": 0.9990688053641733, + "pass@4": 0.9997314116985375, + "pass@8": 0.9999793655718723, + "pass@16": 0.9999999169698948, + "pass@32": 0.9999999999998159, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982886904761905, + "pass@2": 0.9999117297837768, + "pass@4": 0.9999994979794193, + "pass@8": 0.9999999999856092, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9999620591494178, + "pass@4": 0.9999997176625378, + "pass@8": 0.9999999999937114, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9974112426035503, + "pass@2": 0.9990299439733494, + "pass@4": 0.999743682649876, + "pass@8": 0.9999804637850213, + "pass@16": 0.9999999213916163, + "pass@32": 0.9999999999998258, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9911608573717948, + "pass@2": 0.9963473052947709, + "pass@4": 0.9984855673088942, + "pass@8": 0.9994635469329705, + "pass@16": 0.9999228935849399, + "pass@32": 0.9999990402568896, + "pass@64": 0.9999999999948799, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9897447183098591, + "pass@2": 0.9935465648220028, + "pass@4": 0.9950367868629096, + "pass@8": 0.9961759013482411, + "pass@16": 0.9975146715824619, + "pass@32": 0.9990370240116533, + "pass@64": 0.9999221200063837, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9560685198300283, + "pass@2": 0.972025147219558, + "pass@4": 0.9833676431380923, + "pass@8": 0.9902832202553371, + "pass@16": 0.9946957855807873, + "pass@32": 0.997549645536328, + "pass@64": 0.9992147611382037, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.943215925141243, + "pass@2": 0.9660565138573776, + "pass@4": 0.9814934266691235, + "pass@8": 0.9904757233909742, + "pass@16": 0.9957168814771249, + "pass@32": 0.9984134645336938, + "pass@64": 0.9996455904087194, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9108787225705329, + "pass@2": 0.9434656776837063, + "pass@4": 0.9667659061230197, + "pass@8": 0.9808162550485344, + "pass@16": 0.9881944688814267, + "pass@32": 0.9930606223263609, + "pass@64": 0.9966675055170503, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9186380451713395, + "pass@2": 0.9460888249074009, + "pass@4": 0.9653372817299704, + "pass@8": 0.9779994019698924, + "pass@16": 0.9860626693919856, + "pass@32": 0.9919049064530143, + "pass@64": 0.9963014598578678, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9521540823699421, + "pass@2": 0.9741484513677119, + "pass@4": 0.985112245633313, + "pass@8": 0.9908469302823427, + "pass@16": 0.9934400862091718, + "pass@32": 0.994992084796452, + "pass@64": 0.9963530196364373, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8628276209677419, + "pass@2": 0.9018546005842007, + "pass@4": 0.9287450191709906, + "pass@8": 0.9468683936291992, + "pass@16": 0.9613419346632679, + "pass@32": 0.9736111775442221, + "pass@64": 0.9823099408096163, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.884709176300578, + "pass@2": 0.9211279214646582, + "pass@4": 0.9415992674867953, + "pass@8": 0.9565759689885837, + "pass@16": 0.9692118557186207, + "pass@32": 0.9795491000031725, + "pass@64": 0.98639472632689, + "pass@128": 0.9884393063583815 + }, + "18": { + "pass@1": 0.82296875, + "pass@2": 0.8701905230596182, + "pass@4": 0.8998161420001068, + "pass@8": 0.9205166862174053, + "pass@16": 0.9355630508173823, + "pass@32": 0.9477589149914283, + "pass@64": 0.9590373492455058, + "pass@128": 0.9685714285714285 + }, + "19": { + "pass@1": 0.7173188813813813, + "pass@2": 0.7844284983802695, + "pass@4": 0.8315280263390493, + "pass@8": 0.8687886876667537, + "pass@16": 0.8990525439832475, + "pass@32": 0.9216565068026786, + "pass@64": 0.9364336919565242, + "pass@128": 0.9459459459459459 + }, + "20": { + "pass@1": 0.7638419669669669, + "pass@2": 0.8345680818613895, + "pass@4": 0.8795241003095331, + "pass@8": 0.9096289064963395, + "pass@16": 0.9287254882088332, + "pass@32": 0.9412760341681837, + "pass@64": 0.9527062784829503, + "pass@128": 0.963963963963964 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 766904, + "answer_accuracy": 0.933973109898675, + "avg_response_len": 292.6585639127046, + "resp_tokens_sum": 240307800, + "pass_at_k": { + "pass@1": 0.933973109898675, + "pass@2": 0.9541137397892366, + "pass@4": 0.9676575266507484, + "pass@8": 0.9769106752250937, + "pass@16": 0.9831209304949401, + "pass@32": 0.9874264895458542, + "pass@64": 0.9906284085148328, + "pass@128": 0.9929851909586905 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9263564033742331, + "pass@2": 0.943031511158881, + "pass@4": 0.9574623036959338, + "pass@8": 0.9692292659689711, + "pass@16": 0.9785950061939993, + "pass@32": 0.985816650956142, + "pass@64": 0.9927118784293236, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9957859848484848, + "pass@2": 0.9979222590073968, + "pass@4": 0.9992018259081249, + "pass@8": 0.9998472966011496, + "pass@16": 0.999994268588126, + "pass@32": 0.9999999968257123, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996764053254438, + "pass@2": 0.9999919920328006, + "pass@4": 0.999999990293373, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996286231884058, + "pass@2": 0.9990335786830994, + "pass@4": 0.9998567722512948, + "pass@8": 0.9999954258079576, + "pass@16": 0.9999999965927238, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997089899713467, + "pass@2": 0.9999936545360197, + "pass@4": 0.9999999959711339, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9963872832369942, + "pass@2": 0.9973061080515223, + "pass@4": 0.9977670850211643, + "pass@8": 0.9983013813782712, + "pass@16": 0.999038785005553, + "pass@32": 0.9997319443095561, + "pass@64": 0.9999910524153849, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9921654929577465, + "pass@2": 0.9949418459576357, + "pass@4": 0.9969772593566649, + "pass@8": 0.998742979098145, + "pass@16": 0.9996984788969664, + "pass@32": 0.9999800029461254, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932634730538922, + "pass@2": 0.9957594653213259, + "pass@4": 0.9968023138200539, + "pass@8": 0.9971748134078343, + "pass@16": 0.9973801780816433, + "pass@32": 0.9977544910178684, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9698795180722891, + "pass@2": 0.9801018789725835, + "pass@4": 0.9859459858556837, + "pass@8": 0.9903607486554723, + "pass@16": 0.9944354686155655, + "pass@32": 0.997734424893915, + "pass@64": 0.9996912428943647, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9312793887147336, + "pass@2": 0.9578514921136424, + "pass@4": 0.9785946927088653, + "pass@8": 0.9911220322102386, + "pass@16": 0.9969256254585147, + "pass@32": 0.999356859530079, + "pass@64": 0.9999770727927124, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9410014005602241, + "pass@2": 0.9629413281060455, + "pass@4": 0.9783460898795218, + "pass@8": 0.9885164313887403, + "pass@16": 0.9938448297567265, + "pass@32": 0.99557976291731, + "pass@64": 0.9965040977444787, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.938093932748538, + "pass@2": 0.9617224553345302, + "pass@4": 0.9780836096145873, + "pass@8": 0.9891690673671412, + "pass@16": 0.9953444109429628, + "pass@32": 0.9985866789814201, + "pass@64": 0.9998999204723729, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.946875, + "pass@2": 0.9708445180147938, + "pass@4": 0.9852852896228875, + "pass@8": 0.9932527012437333, + "pass@16": 0.9965859651004563, + "pass@32": 0.9976875706827749, + "pass@64": 0.9984848257468957, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8979255698005698, + "pass@2": 0.9284205560042177, + "pass@4": 0.9476878840999577, + "pass@8": 0.9601126751922175, + "pass@16": 0.9681501715762721, + "pass@32": 0.9745545081734679, + "pass@64": 0.9806737994273104, + "pass@128": 0.9857549857549858 + }, + "17": { + "pass@1": 0.883103649068323, + "pass@2": 0.9193444484520957, + "pass@4": 0.9431839631265031, + "pass@8": 0.9588158770909269, + "pass@16": 0.969734921800441, + "pass@32": 0.9777583996276256, + "pass@64": 0.9834804448754739, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8681985294117647, + "pass@2": 0.9134885942566008, + "pass@4": 0.9388149804803808, + "pass@8": 0.956182315324883, + "pass@16": 0.969199828469146, + "pass@32": 0.9786850953286746, + "pass@64": 0.98495366662752, + "pass@128": 0.9882352941176471 + }, + "19": { + "pass@1": 0.7562779017857143, + "pass@2": 0.8173970929415075, + "pass@4": 0.8586776217704924, + "pass@8": 0.8873359588818277, + "pass@16": 0.9093487259281797, + "pass@32": 0.9286139494295144, + "pass@64": 0.9465647885804772, + "pass@128": 0.9613095238095238 + }, + "20": { + "pass@1": 0.7049962574850299, + "pass@2": 0.7821469511763871, + "pass@4": 0.8387314392587147, + "pass@8": 0.8803005442248173, + "pass@16": 0.9090276962957424, + "pass@32": 0.927911101416634, + "pass@64": 0.9395962270441276, + "pass@128": 0.9461077844311377 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 744763, + "answer_accuracy": 0.9296151042498801, + "avg_response_len": 313.71964995406614, + "resp_tokens_sum": 251337125, + "pass_at_k": { + "pass@1": 0.9296151042498801, + "pass@2": 0.951017935747319, + "pass@4": 0.9651440403059918, + "pass@8": 0.974651918770272, + "pass@16": 0.9810899167866008, + "pass@32": 0.9857287259116158, + "pass@64": 0.9891995583526448, + "pass@128": 0.9915321936411567 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9518889925373134, + "pass@2": 0.9682519978845928, + "pass@4": 0.9806292692891003, + "pass@8": 0.9888008490677124, + "pass@16": 0.9928884788352844, + "pass@32": 0.9939672147924427, + "pass@64": 0.9940297978227394, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999960543835081, + "pass@4": 0.9999999986335526, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992233187134503, + "pass@2": 0.9998215683565871, + "pass@4": 0.9999901437758876, + "pass@8": 0.9999999784883499, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940946691176471, + "pass@2": 0.9972234686197314, + "pass@4": 0.9979508072520346, + "pass@8": 0.9985971409541529, + "pass@16": 0.9993657240569218, + "pass@32": 0.999893411003342, + "pass@64": 0.9999991012534062, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998767744479495, + "pass@2": 0.9999976713281503, + "pass@4": 0.9999999997042957, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9922122231012658, + "pass@2": 0.9943055105651352, + "pass@4": 0.9964102716986326, + "pass@8": 0.9984351822242498, + "pass@16": 0.9996795269701011, + "pass@32": 0.9999902592414398, + "pass@64": 0.9999999992697213, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930555555555556, + "pass@2": 0.9965503150788586, + "pass@4": 0.9984686342698152, + "pass@8": 0.9996429053090278, + "pass@16": 0.9999814246833897, + "pass@32": 0.999999972034264, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9924135852090032, + "pass@2": 0.9953062099653139, + "pass@4": 0.9966570869758643, + "pass@8": 0.9975856061587218, + "pass@16": 0.9983673492925828, + "pass@32": 0.9992571222626693, + "pass@64": 0.9999073347342836, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9599702380952381, + "pass@2": 0.9765876921634795, + "pass@4": 0.989391186815934, + "pass@8": 0.9963180825233957, + "pass@16": 0.9992307852616401, + "pass@32": 0.9999490539138667, + "pass@64": 0.9999999121061375, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.939865252293578, + "pass@2": 0.9603624124106047, + "pass@4": 0.9760637159460566, + "pass@8": 0.9866032202849236, + "pass@16": 0.9926096934140741, + "pass@32": 0.9959497775979793, + "pass@64": 0.9984405444345842, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9073832947530864, + "pass@2": 0.9443962191358023, + "pass@4": 0.9685251580242282, + "pass@8": 0.9828957795842923, + "pass@16": 0.9915318725469653, + "pass@32": 0.996952650279957, + "pass@64": 0.9995923226260632, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9464252596439169, + "pass@2": 0.9695290777821917, + "pass@4": 0.9824109808974172, + "pass@8": 0.9893288765335709, + "pass@16": 0.9930767880716179, + "pass@32": 0.9956401539621311, + "pass@64": 0.996931099741985, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9288194444444444, + "pass@2": 0.9530141197628071, + "pass@4": 0.9679944280691763, + "pass@8": 0.9773730797152483, + "pass@16": 0.9826136664666306, + "pass@32": 0.9854947655239579, + "pass@64": 0.9876295766722463, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.8941049041297935, + "pass@2": 0.932356267275219, + "pass@4": 0.9575642494356341, + "pass@8": 0.9720565681989967, + "pass@16": 0.9788168570235897, + "pass@32": 0.9821212614436396, + "pass@64": 0.9848713086920479, + "pass@128": 0.9882005899705014 + }, + "17": { + "pass@1": 0.8530685240963856, + "pass@2": 0.8834865791907793, + "pass@4": 0.9036101018246212, + "pass@8": 0.9202718219997545, + "pass@16": 0.9349628711134209, + "pass@32": 0.9486017624674481, + "pass@64": 0.9615355997566628, + "pass@128": 0.9728915662650602 + }, + "18": { + "pass@1": 0.8661290322580645, + "pass@2": 0.9098425196850394, + "pass@4": 0.9353601161749935, + "pass@8": 0.9524239545873169, + "pass@16": 0.9650574815675628, + "pass@32": 0.9742872492835252, + "pass@64": 0.9800047314029565, + "pass@128": 0.9838709677419355 + }, + "19": { + "pass@1": 0.7272233761329305, + "pass@2": 0.7992720698432335, + "pass@4": 0.8498617207403449, + "pass@8": 0.886109822147316, + "pass@16": 0.9119255337966973, + "pass@32": 0.9312105063485466, + "pass@64": 0.946458090289178, + "pass@128": 0.9546827794561934 + }, + "20": { + "pass@1": 0.714573948948949, + "pass@2": 0.7877462846704968, + "pass@4": 0.8363218603868204, + "pass@8": 0.8718850949032955, + "pass@16": 0.9008473092143473, + "pass@32": 0.9259147136391661, + "pass@64": 0.945745298634043, + "pass@128": 0.9579579579579579 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2bd737bf9e23b7521ad93c758230e229ada51dbb --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71bc0a4156c3b115a52f46d0b7a4f3618bf23401f2e6767e79f8902ca05e3eba +size 2725879383 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a96406b4023760819010fcf968a98ebc0b65ee53 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/graphs", "generated_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive", "results_dir": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1021882, "value_mismatch": 12712, "dependency_mismatch": 127940, "answer_mismatch": 11417, "extra_nodes": 14347, "missing_nodes": 9553, "perfect_with_extra": 2436, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 336755, "value_mismatch": 4456, "answer_mismatch": 3891, "dependency_mismatch": 41386, "extra_nodes": 4330, "perfect_with_extra": 701, "missing_nodes": 2877}, "teachers_in_school": {"total": 390656, "perfect": 345996, "extra_nodes": 5099, "perfect_with_extra": 790, "value_mismatch": 4496, "dependency_mismatch": 44048, "answer_mismatch": 4246, "missing_nodes": 3816}, "movie_festival_awards": {"total": 382336, "perfect": 339131, "dependency_mismatch": 42506, "value_mismatch": 3760, "answer_mismatch": 3280, "extra_nodes": 4918, "perfect_with_extra": 945, "missing_nodes": 2860}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 652944, "value_mismatch": 209592, "dependency_mismatch": 582793, "answer_mismatch": 157333, "extra_nodes": 48569, "missing_nodes": 97545, "perfect_with_extra": 581, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 219461, "dependency_mismatch": 195621, "value_mismatch": 71003, "answer_mismatch": 54254, "extra_nodes": 19010, "missing_nodes": 33996, "perfect_with_extra": 221}, "movie_festival_awards": {"total": 418816, "perfect": 213304, "dependency_mismatch": 189668, "extra_nodes": 14544, "missing_nodes": 31174, "value_mismatch": 70774, "answer_mismatch": 53109, "perfect_with_extra": 138}, "teachers_in_school": {"total": 430464, "perfect": 220179, "value_mismatch": 67815, "dependency_mismatch": 197504, "answer_mismatch": 49970, "extra_nodes": 15015, "perfect_with_extra": 222, "missing_nodes": 32375}}, "results_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1674826, "value_mismatch": 222304, "dependency_mismatch": 710733, "answer_mismatch": 168750, "extra_nodes": 62916, "missing_nodes": 107098, "perfect_with_extra": 3017, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 556216, "value_mismatch": 75459, "answer_mismatch": 58145, "dependency_mismatch": 237007, "extra_nodes": 23340, "perfect_with_extra": 922, "missing_nodes": 36873}, "teachers_in_school": {"total": 821120, "perfect": 566175, "extra_nodes": 20114, "perfect_with_extra": 1012, "value_mismatch": 72311, "dependency_mismatch": 241552, "answer_mismatch": 54216, "missing_nodes": 36191}, "movie_festival_awards": {"total": 801152, "perfect": 552435, "dependency_mismatch": 232174, "value_mismatch": 74534, "answer_mismatch": 56389, "extra_nodes": 19462, "perfect_with_extra": 1083, "missing_nodes": 34034}}}} diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8f38c9e3461fbccfef42ff876a15039b26a85d12 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4055115d0f5e5fb2c491be9fa5d91e947da241290ddc1c5ce7b63399eb3c601 +size 373327912 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3f148aed16ab45804289af024a809001c91a440b --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d5e2d06aebf6bab2d8d3e2709c564668359fa2bc57db636d078e24f8cdce31b +size 421915420 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/results.json b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..fd9c839f8c411523506c51aed35ea9c6985e6d74 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-39470_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 0.9969418960244648, + "pass@4": 0.9969418960244648, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.99375, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9746031746031746, + "pass@2": 0.9936507936507937, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9610778443113772, + "pass@2": 0.9760479041916168, + "pass@4": 0.9910179640718563, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8846153846153846, + "pass@2": 0.9289940828402367, + "pass@4": 0.9674556213017751, + "pass@8": 0.985207100591716, + "pass@16": 0.9940828402366864, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8076923076923077, + "pass@2": 0.875, + "pass@4": 0.9358974358974359, + "pass@8": 0.9775641025641025, + "pass@16": 0.9935897435897436, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7661971830985915, + "pass@2": 0.847887323943662, + "pass@4": 0.9126760563380282, + "pass@8": 0.9605633802816902, + "pass@16": 0.9943661971830986, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6283185840707964, + "pass@2": 0.6696165191740413, + "pass@4": 0.7787610619469026, + "pass@8": 0.8643067846607669, + "pass@16": 0.8967551622418879, + "pass@32": 0.9262536873156342, + "pass@64": 0.9469026548672567, + "pass@128": 0.9528023598820059 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.990625, + "5": 0.9746031746031746, + "6": 0.9610778443113772, + "7": 0.8846153846153846, + "8": 0.8076923076923077, + "9": 0.7661971830985915, + "10": 0.6283185840707964 + }, + "pass@2": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.99375, + "5": 0.9936507936507937, + "6": 0.9760479041916168, + "7": 0.9289940828402367, + "8": 0.875, + "9": 0.847887323943662, + "10": 0.6696165191740413 + }, + "pass@4": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 1.0, + "5": 1.0, + "6": 0.9910179640718563, + "7": 0.9674556213017751, + "8": 0.9358974358974359, + "9": 0.9126760563380282, + "10": 0.7787610619469026 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9940119760479041, + "7": 0.985207100591716, + "8": 0.9775641025641025, + "9": 0.9605633802816902, + "10": 0.8643067846607669 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.8967551622418879 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9262536873156342 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9469026548672567 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9528023598820059 + } + }, + "total_pass_at_k": { + "pass@1": 0.8875379939209727, + "pass@2": 0.9182708544410672, + "pass@4": 0.9523809523809523, + "pass@8": 0.9750084430935495, + "pass@16": 0.9858156028368794, + "pass@32": 0.9912191827085444, + "pass@64": 0.993920972644377, + "pass@128": 0.994596420128335 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9970760233918129, + "pass@2": 0.9970760233918129, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.961764705882353, + "pass@2": 0.9764705882352941, + "pass@4": 0.9941176470588236, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9684542586750788, + "pass@2": 0.9842271293375394, + "pass@4": 0.9936908517350158, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8544303797468354, + "pass@2": 0.8987341772151899, + "pass@4": 0.9525316455696202, + "pass@8": 0.9715189873417721, + "pass@16": 0.9936708860759493, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.8138138138138138, + "pass@2": 0.8648648648648649, + "pass@4": 0.93993993993994, + "pass@8": 0.975975975975976, + "pass@16": 0.993993993993994, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7491961414790996, + "pass@2": 0.8135048231511254, + "pass@4": 0.8745980707395499, + "pass@8": 0.932475884244373, + "pass@16": 0.9614147909967846, + "pass@32": 0.9967845659163987, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "10": { + "pass@1": 0.6238805970149254, + "pass@2": 0.6895522388059702, + "pass@4": 0.7611940298507462, + "pass@8": 0.8417910447761194, + "pass@16": 0.8955223880597015, + "pass@32": 0.9313432835820895, + "pass@64": 0.9522388059701492, + "pass@128": 0.9731343283582089 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.961764705882353, + "6": 0.9684542586750788, + "7": 0.8544303797468354, + "8": 0.8138138138138138, + "9": 0.7491961414790996, + "10": 0.6238805970149254 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9764705882352941, + "6": 0.9842271293375394, + "7": 0.8987341772151899, + "8": 0.8648648648648649, + "9": 0.8135048231511254, + "10": 0.6895522388059702 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 0.9936908517350158, + "7": 0.9525316455696202, + "8": 0.93993993993994, + "9": 0.8745980707395499, + "10": 0.7611940298507462 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.9715189873417721, + "8": 0.975975975975976, + "9": 0.932475884244373, + "10": 0.8417910447761194 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9614147909967846, + "10": 0.8955223880597015 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9313432835820895 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9522388059701492 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9731343283582089 + } + }, + "total_pass_at_k": { + "pass@1": 0.8875125544024104, + "pass@2": 0.9152996317375293, + "pass@4": 0.9471041178439906, + "pass@8": 0.9688650820220958, + "pass@16": 0.9825912286575159, + "pass@32": 0.9912956143287579, + "pass@64": 0.9936391027787077, + "pass@128": 0.9959825912286575 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990909090909091, + "pass@2": 0.996969696969697, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9881656804733728, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9797101449275363, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9742120343839542, + "pass@2": 0.9770773638968482, + "pass@4": 0.9828080229226361, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757225433526011, + "pass@2": 0.930635838150289, + "pass@4": 0.9653179190751445, + "pass@8": 0.9797687861271677, + "pass@16": 0.9942196531791907, + "pass@32": 0.9971098265895953, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8, + "pass@2": 0.8788732394366198, + "pass@4": 0.9408450704225352, + "pass@8": 0.9746478873239437, + "pass@16": 0.9971830985915493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7215568862275449, + "pass@2": 0.7964071856287425, + "pass@4": 0.9011976047904192, + "pass@8": 0.9461077844311377, + "pass@16": 0.9790419161676647, + "pass@32": 0.9970059880239521, + "pass@64": 0.9970059880239521, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6687116564417178, + "pass@2": 0.7085889570552147, + "pass@4": 0.7822085889570553, + "pass@8": 0.8650306748466258, + "pass@16": 0.8895705521472392, + "pass@32": 0.9325153374233128, + "pass@64": 0.9478527607361963, + "pass@128": 0.950920245398773 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.990909090909091, + "4": 0.985207100591716, + "5": 0.9536231884057971, + "6": 0.9742120343839542, + "7": 0.8757225433526011, + "8": 0.8, + "9": 0.7215568862275449, + "10": 0.6687116564417178 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9881656804733728, + "5": 0.9797101449275363, + "6": 0.9770773638968482, + "7": 0.930635838150289, + "8": 0.8788732394366198, + "9": 0.7964071856287425, + "10": 0.7085889570552147 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 1.0, + "6": 0.9828080229226361, + "7": 0.9653179190751445, + "8": 0.9408450704225352, + "9": 0.9011976047904192, + "10": 0.7822085889570553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9797687861271677, + "8": 0.9746478873239437, + "9": 0.9461077844311377, + "10": 0.8650306748466258 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9942196531791907, + "8": 0.9971830985915493, + "9": 0.9790419161676647, + "10": 0.8895705521472392 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9325153374233128 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9478527607361963 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.950920245398773 + } + }, + "total_pass_at_k": { + "pass@1": 0.8859764089121888, + "pass@2": 0.918086500655308, + "pass@4": 0.9528178243774574, + "pass@8": 0.9744429882044561, + "pass@16": 0.9849279161205766, + "pass@32": 0.9921363040629095, + "pass@64": 0.9941022280471822, + "pass@128": 0.9947575360419397 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996, + "pass@2": 0.998, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.991, + "pass@2": 0.993, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.963, + "pass@2": 0.983, + "pass@4": 0.998, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.968, + "pass@2": 0.979, + "pass@4": 0.989, + "pass@8": 0.997, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.872, + "pass@2": 0.92, + "pass@4": 0.962, + "pass@8": 0.979, + "pass@16": 0.994, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.807, + "pass@2": 0.873, + "pass@4": 0.939, + "pass@8": 0.976, + "pass@16": 0.995, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.746, + "pass@2": 0.82, + "pass@4": 0.897, + "pass@8": 0.947, + "pass@16": 0.979, + "pass@32": 0.997, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.64, + "pass@2": 0.689, + "pass@4": 0.774, + "pass@8": 0.857, + "pass@16": 0.894, + "pass@32": 0.93, + "pass@64": 0.949, + "pass@128": 0.959 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996, + "4": 0.991, + "5": 0.963, + "6": 0.968, + "7": 0.872, + "8": 0.807, + "9": 0.746, + "10": 0.64 + }, + "pass@2": { + "2": 1.0, + "3": 0.998, + "4": 0.993, + "5": 0.983, + "6": 0.979, + "7": 0.92, + "8": 0.873, + "9": 0.82, + "10": 0.689 + }, + "pass@4": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.998, + "6": 0.989, + "7": 0.962, + "8": 0.939, + "9": 0.897, + "10": 0.774 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.997, + "7": 0.979, + "8": 0.976, + "9": 0.947, + "10": 0.857 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.999, + "7": 0.994, + "8": 0.995, + "9": 0.979, + "10": 0.894 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.998, + "8": 1.0, + "9": 0.997, + "10": 0.93 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 0.998, + "10": 0.949 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 0.999, + "10": 0.959 + } + }, + "total_pass_at_k": { + "pass@1": 0.887, + "pass@2": 0.9172222222222223, + "pass@4": 0.9507777777777778, + "pass@8": 0.9727777777777777, + "pass@16": 0.9844444444444445, + "pass@32": 0.9915555555555555, + "pass@64": 0.9938888888888889, + "pass@128": 0.9951111111111111 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-39470_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5864022662889519, + "pass@2": 0.660056657223796, + "pass@4": 0.7450424929178471, + "pass@8": 0.8498583569405099, + "pass@16": 0.9291784702549575, + "pass@32": 0.9631728045325779, + "pass@64": 0.9801699716713881, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.632768361581921, + "pass@2": 0.6836158192090396, + "pass@4": 0.7711864406779662, + "pass@8": 0.8531073446327684, + "pass@16": 0.9011299435028248, + "pass@32": 0.9491525423728814, + "pass@64": 0.963276836158192, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.5862068965517241, + "pass@2": 0.6551724137931034, + "pass@4": 0.7115987460815048, + "pass@8": 0.7899686520376176, + "pass@16": 0.8432601880877743, + "pass@32": 0.9028213166144201, + "pass@64": 0.9404388714733543, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.5327102803738317, + "pass@2": 0.5887850467289719, + "pass@4": 0.632398753894081, + "pass@8": 0.6666666666666666, + "pass@16": 0.7507788161993769, + "pass@32": 0.822429906542056, + "pass@64": 0.881619937694704, + "pass@128": 0.9190031152647975 + }, + "15": { + "pass@1": 0.5028901734104047, + "pass@2": 0.5375722543352601, + "pass@4": 0.5867052023121387, + "pass@8": 0.6445086705202312, + "pass@16": 0.7167630057803468, + "pass@32": 0.8005780346820809, + "pass@64": 0.8583815028901735, + "pass@128": 0.9190751445086706 + }, + "16": { + "pass@1": 0.4774193548387097, + "pass@2": 0.5096774193548387, + "pass@4": 0.5451612903225806, + "pass@8": 0.5935483870967742, + "pass@16": 0.6387096774193548, + "pass@32": 0.6774193548387096, + "pass@64": 0.7451612903225806, + "pass@128": 0.7935483870967742 + }, + "17": { + "pass@1": 0.523121387283237, + "pass@2": 0.546242774566474, + "pass@4": 0.5722543352601156, + "pass@8": 0.5924855491329479, + "pass@16": 0.6445086705202312, + "pass@32": 0.7138728323699421, + "pass@64": 0.7861271676300579, + "pass@128": 0.8323699421965318 + }, + "18": { + "pass@1": 0.44285714285714284, + "pass@2": 0.4942857142857143, + "pass@4": 0.52, + "pass@8": 0.5457142857142857, + "pass@16": 0.5942857142857143, + "pass@32": 0.6485714285714286, + "pass@64": 0.7171428571428572, + "pass@128": 0.7742857142857142 + }, + "19": { + "pass@1": 0.3843843843843844, + "pass@2": 0.4144144144144144, + "pass@4": 0.4444444444444444, + "pass@8": 0.4984984984984985, + "pass@16": 0.5375375375375375, + "pass@32": 0.5795795795795796, + "pass@64": 0.6486486486486487, + "pass@128": 0.7147147147147147 + }, + "20": { + "pass@1": 0.3963963963963964, + "pass@2": 0.42342342342342343, + "pass@4": 0.4744744744744745, + "pass@8": 0.5045045045045045, + "pass@16": 0.5705705705705706, + "pass@32": 0.6156156156156156, + "pass@64": 0.6606606606606606, + "pass@128": 0.7207207207207207 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5864022662889519, + "12": 0.632768361581921, + "13": 0.5862068965517241, + "14": 0.5327102803738317, + "15": 0.5028901734104047, + "16": 0.4774193548387097, + "17": 0.523121387283237, + "18": 0.44285714285714284, + "19": 0.3843843843843844, + "20": 0.3963963963963964 + }, + "pass@2": { + "11": 0.660056657223796, + "12": 0.6836158192090396, + "13": 0.6551724137931034, + "14": 0.5887850467289719, + "15": 0.5375722543352601, + "16": 0.5096774193548387, + "17": 0.546242774566474, + "18": 0.4942857142857143, + "19": 0.4144144144144144, + "20": 0.42342342342342343 + }, + "pass@4": { + "11": 0.7450424929178471, + "12": 0.7711864406779662, + "13": 0.7115987460815048, + "14": 0.632398753894081, + "15": 0.5867052023121387, + "16": 0.5451612903225806, + "17": 0.5722543352601156, + "18": 0.52, + "19": 0.4444444444444444, + "20": 0.4744744744744745 + }, + "pass@8": { + "11": 0.8498583569405099, + "12": 0.8531073446327684, + "13": 0.7899686520376176, + "14": 0.6666666666666666, + "15": 0.6445086705202312, + "16": 0.5935483870967742, + "17": 0.5924855491329479, + "18": 0.5457142857142857, + "19": 0.4984984984984985, + "20": 0.5045045045045045 + }, + "pass@16": { + "11": 0.9291784702549575, + "12": 0.9011299435028248, + "13": 0.8432601880877743, + "14": 0.7507788161993769, + "15": 0.7167630057803468, + "16": 0.6387096774193548, + "17": 0.6445086705202312, + "18": 0.5942857142857143, + "19": 0.5375375375375375, + "20": 0.5705705705705706 + }, + "pass@32": { + "11": 0.9631728045325779, + "12": 0.9491525423728814, + "13": 0.9028213166144201, + "14": 0.822429906542056, + "15": 0.8005780346820809, + "16": 0.6774193548387096, + "17": 0.7138728323699421, + "18": 0.6485714285714286, + "19": 0.5795795795795796, + "20": 0.6156156156156156 + }, + "pass@64": { + "11": 0.9801699716713881, + "12": 0.963276836158192, + "13": 0.9404388714733543, + "14": 0.881619937694704, + "15": 0.8583815028901735, + "16": 0.7451612903225806, + "17": 0.7861271676300579, + "18": 0.7171428571428572, + "19": 0.6486486486486487, + "20": 0.6606606606606606 + }, + "pass@128": { + "11": 0.9858356940509915, + "12": 0.9745762711864406, + "13": 0.9592476489028213, + "14": 0.9190031152647975, + "15": 0.9190751445086706, + "16": 0.7935483870967742, + "17": 0.8323699421965318, + "18": 0.7742857142857142, + "19": 0.7147147147147147, + "20": 0.7207207207207207 + } + }, + "total_pass_at_k": { + "pass@1": 0.5072808320950966, + "pass@2": 0.5521545319465082, + "pass@4": 0.6014858841010401, + "pass@8": 0.6552748885586924, + "pass@16": 0.7141158989598811, + "pass@32": 0.7687964338781575, + "pass@64": 0.8193164933135215, + "pass@128": 0.8603268945022289 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5746031746031746, + "pass@2": 0.6444444444444445, + "pass@4": 0.7619047619047619, + "pass@8": 0.8761904761904762, + "pass@16": 0.9396825396825397, + "pass@32": 0.9746031746031746, + "pass@64": 0.9873015873015873, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.6299694189602446, + "pass@2": 0.6788990825688074, + "pass@4": 0.7737003058103975, + "pass@8": 0.8685015290519877, + "pass@16": 0.9143730886850153, + "pass@32": 0.9541284403669725, + "pass@64": 0.9663608562691132, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.5833333333333334, + "pass@2": 0.6141975308641975, + "pass@4": 0.6851851851851852, + "pass@8": 0.7777777777777778, + "pass@16": 0.845679012345679, + "pass@32": 0.904320987654321, + "pass@64": 0.9320987654320988, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.5548961424332344, + "pass@2": 0.5934718100890207, + "pass@4": 0.6261127596439169, + "pass@8": 0.7002967359050445, + "pass@16": 0.7833827893175074, + "pass@32": 0.8664688427299704, + "pass@64": 0.9169139465875371, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.4876543209876543, + "pass@2": 0.5277777777777778, + "pass@4": 0.5493827160493827, + "pass@8": 0.5987654320987654, + "pass@16": 0.691358024691358, + "pass@32": 0.7870370370370371, + "pass@64": 0.8672839506172839, + "pass@128": 0.9166666666666666 + }, + "16": { + "pass@1": 0.5398230088495575, + "pass@2": 0.5781710914454278, + "pass@4": 0.6253687315634219, + "pass@8": 0.6755162241887905, + "pass@16": 0.7404129793510325, + "pass@32": 0.7876106194690266, + "pass@64": 0.8259587020648967, + "pass@128": 0.8466076696165191 + }, + "17": { + "pass@1": 0.4789156626506024, + "pass@2": 0.5090361445783133, + "pass@4": 0.5301204819277109, + "pass@8": 0.5692771084337349, + "pass@16": 0.6234939759036144, + "pass@32": 0.6837349397590361, + "pass@64": 0.7439759036144579, + "pass@128": 0.7951807228915663 + }, + "18": { + "pass@1": 0.4645161290322581, + "pass@2": 0.5161290322580645, + "pass@4": 0.535483870967742, + "pass@8": 0.5806451612903226, + "pass@16": 0.6161290322580645, + "pass@32": 0.6741935483870968, + "pass@64": 0.7451612903225806, + "pass@128": 0.8225806451612904 + }, + "19": { + "pass@1": 0.38972809667673713, + "pass@2": 0.4259818731117825, + "pass@4": 0.4652567975830816, + "pass@8": 0.513595166163142, + "pass@16": 0.5619335347432024, + "pass@32": 0.6042296072507553, + "pass@64": 0.6586102719033232, + "pass@128": 0.7039274924471299 + }, + "20": { + "pass@1": 0.36036036036036034, + "pass@2": 0.3933933933933934, + "pass@4": 0.4144144144144144, + "pass@8": 0.45645645645645644, + "pass@16": 0.4864864864864865, + "pass@32": 0.5375375375375375, + "pass@64": 0.6036036036036037, + "pass@128": 0.6666666666666666 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5746031746031746, + "12": 0.6299694189602446, + "13": 0.5833333333333334, + "14": 0.5548961424332344, + "15": 0.4876543209876543, + "16": 0.5398230088495575, + "17": 0.4789156626506024, + "18": 0.4645161290322581, + "19": 0.38972809667673713, + "20": 0.36036036036036034 + }, + "pass@2": { + "11": 0.6444444444444445, + "12": 0.6788990825688074, + "13": 0.6141975308641975, + "14": 0.5934718100890207, + "15": 0.5277777777777778, + "16": 0.5781710914454278, + "17": 0.5090361445783133, + "18": 0.5161290322580645, + "19": 0.4259818731117825, + "20": 0.3933933933933934 + }, + "pass@4": { + "11": 0.7619047619047619, + "12": 0.7737003058103975, + "13": 0.6851851851851852, + "14": 0.6261127596439169, + "15": 0.5493827160493827, + "16": 0.6253687315634219, + "17": 0.5301204819277109, + "18": 0.535483870967742, + "19": 0.4652567975830816, + "20": 0.4144144144144144 + }, + "pass@8": { + "11": 0.8761904761904762, + "12": 0.8685015290519877, + "13": 0.7777777777777778, + "14": 0.7002967359050445, + "15": 0.5987654320987654, + "16": 0.6755162241887905, + "17": 0.5692771084337349, + "18": 0.5806451612903226, + "19": 0.513595166163142, + "20": 0.45645645645645644 + }, + "pass@16": { + "11": 0.9396825396825397, + "12": 0.9143730886850153, + "13": 0.845679012345679, + "14": 0.7833827893175074, + "15": 0.691358024691358, + "16": 0.7404129793510325, + "17": 0.6234939759036144, + "18": 0.6161290322580645, + "19": 0.5619335347432024, + "20": 0.4864864864864865 + }, + "pass@32": { + "11": 0.9746031746031746, + "12": 0.9541284403669725, + "13": 0.904320987654321, + "14": 0.8664688427299704, + "15": 0.7870370370370371, + "16": 0.7876106194690266, + "17": 0.6837349397590361, + "18": 0.6741935483870968, + "19": 0.6042296072507553, + "20": 0.5375375375375375 + }, + "pass@64": { + "11": 0.9873015873015873, + "12": 0.9663608562691132, + "13": 0.9320987654320988, + "14": 0.9169139465875371, + "15": 0.8672839506172839, + "16": 0.8259587020648967, + "17": 0.7439759036144579, + "18": 0.7451612903225806, + "19": 0.6586102719033232, + "20": 0.6036036036036037 + }, + "pass@128": { + "11": 0.9904761904761905, + "12": 0.9785932721712538, + "13": 0.9537037037037037, + "14": 0.9465875370919882, + "15": 0.9166666666666666, + "16": 0.8466076696165191, + "17": 0.7951807228915663, + "18": 0.8225806451612904, + "19": 0.7039274924471299, + "20": 0.6666666666666666 + } + }, + "total_pass_at_k": { + "pass@1": 0.5061124694376528, + "pass@2": 0.5476772616136919, + "pass@4": 0.5959657701711492, + "pass@8": 0.660757946210269, + "pass@16": 0.719437652811736, + "pass@32": 0.7765892420537898, + "pass@64": 0.823960880195599, + "pass@128": 0.8612469437652812 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.641566265060241, + "pass@2": 0.7168674698795181, + "pass@4": 0.7831325301204819, + "pass@8": 0.8373493975903614, + "pass@16": 0.9126506024096386, + "pass@32": 0.9518072289156626, + "pass@64": 0.9698795180722891, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.554858934169279, + "pass@2": 0.6426332288401254, + "pass@4": 0.7241379310344828, + "pass@8": 0.8213166144200627, + "pass@16": 0.890282131661442, + "pass@32": 0.9341692789968652, + "pass@64": 0.9623824451410659, + "pass@128": 0.9780564263322884 + }, + "13": { + "pass@1": 0.6218487394957983, + "pass@2": 0.6722689075630253, + "pass@4": 0.7394957983193278, + "pass@8": 0.8151260504201681, + "pass@16": 0.8739495798319328, + "pass@32": 0.9103641456582633, + "pass@64": 0.9635854341736695, + "pass@128": 0.9803921568627451 + }, + "14": { + "pass@1": 0.5467836257309941, + "pass@2": 0.5906432748538012, + "pass@4": 0.6374269005847953, + "pass@8": 0.7309941520467836, + "pass@16": 0.8011695906432749, + "pass@32": 0.8567251461988304, + "pass@64": 0.9035087719298246, + "pass@128": 0.9444444444444444 + }, + "15": { + "pass@1": 0.5484848484848485, + "pass@2": 0.5848484848484848, + "pass@4": 0.6303030303030303, + "pass@8": 0.696969696969697, + "pass@16": 0.7393939393939394, + "pass@32": 0.8242424242424242, + "pass@64": 0.9, + "pass@128": 0.9303030303030303 + }, + "16": { + "pass@1": 0.5327635327635327, + "pass@2": 0.5470085470085471, + "pass@4": 0.5954415954415955, + "pass@8": 0.6210826210826211, + "pass@16": 0.7008547008547008, + "pass@32": 0.7749287749287749, + "pass@64": 0.8091168091168092, + "pass@128": 0.8290598290598291 + }, + "17": { + "pass@1": 0.40683229813664595, + "pass@2": 0.43478260869565216, + "pass@4": 0.4720496894409938, + "pass@8": 0.562111801242236, + "pass@16": 0.6366459627329193, + "pass@32": 0.7018633540372671, + "pass@64": 0.7732919254658385, + "pass@128": 0.8136645962732919 + }, + "18": { + "pass@1": 0.4441176470588235, + "pass@2": 0.4852941176470588, + "pass@4": 0.5117647058823529, + "pass@8": 0.538235294117647, + "pass@16": 0.5941176470588235, + "pass@32": 0.6411764705882353, + "pass@64": 0.7029411764705882, + "pass@128": 0.7823529411764706 + }, + "19": { + "pass@1": 0.42857142857142855, + "pass@2": 0.46130952380952384, + "pass@4": 0.4880952380952381, + "pass@8": 0.5089285714285714, + "pass@16": 0.5386904761904762, + "pass@32": 0.6130952380952381, + "pass@64": 0.6458333333333334, + "pass@128": 0.7053571428571429 + }, + "20": { + "pass@1": 0.38023952095808383, + "pass@2": 0.41317365269461076, + "pass@4": 0.44011976047904194, + "pass@8": 0.4940119760479042, + "pass@16": 0.5209580838323353, + "pass@32": 0.5568862275449101, + "pass@64": 0.625748502994012, + "pass@128": 0.655688622754491 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.641566265060241, + "12": 0.554858934169279, + "13": 0.6218487394957983, + "14": 0.5467836257309941, + "15": 0.5484848484848485, + "16": 0.5327635327635327, + "17": 0.40683229813664595, + "18": 0.4441176470588235, + "19": 0.42857142857142855, + "20": 0.38023952095808383 + }, + "pass@2": { + "11": 0.7168674698795181, + "12": 0.6426332288401254, + "13": 0.6722689075630253, + "14": 0.5906432748538012, + "15": 0.5848484848484848, + "16": 0.5470085470085471, + "17": 0.43478260869565216, + "18": 0.4852941176470588, + "19": 0.46130952380952384, + "20": 0.41317365269461076 + }, + "pass@4": { + "11": 0.7831325301204819, + "12": 0.7241379310344828, + "13": 0.7394957983193278, + "14": 0.6374269005847953, + "15": 0.6303030303030303, + "16": 0.5954415954415955, + "17": 0.4720496894409938, + "18": 0.5117647058823529, + "19": 0.4880952380952381, + "20": 0.44011976047904194 + }, + "pass@8": { + "11": 0.8373493975903614, + "12": 0.8213166144200627, + "13": 0.8151260504201681, + "14": 0.7309941520467836, + "15": 0.696969696969697, + "16": 0.6210826210826211, + "17": 0.562111801242236, + "18": 0.538235294117647, + "19": 0.5089285714285714, + "20": 0.4940119760479042 + }, + "pass@16": { + "11": 0.9126506024096386, + "12": 0.890282131661442, + "13": 0.8739495798319328, + "14": 0.8011695906432749, + "15": 0.7393939393939394, + "16": 0.7008547008547008, + "17": 0.6366459627329193, + "18": 0.5941176470588235, + "19": 0.5386904761904762, + "20": 0.5209580838323353 + }, + "pass@32": { + "11": 0.9518072289156626, + "12": 0.9341692789968652, + "13": 0.9103641456582633, + "14": 0.8567251461988304, + "15": 0.8242424242424242, + "16": 0.7749287749287749, + "17": 0.7018633540372671, + "18": 0.6411764705882353, + "19": 0.6130952380952381, + "20": 0.5568862275449101 + }, + "pass@64": { + "11": 0.9698795180722891, + "12": 0.9623824451410659, + "13": 0.9635854341736695, + "14": 0.9035087719298246, + "15": 0.9, + "16": 0.8091168091168092, + "17": 0.7732919254658385, + "18": 0.7029411764705882, + "19": 0.6458333333333334, + "20": 0.625748502994012 + }, + "pass@128": { + "11": 0.9819277108433735, + "12": 0.9780564263322884, + "13": 0.9803921568627451, + "14": 0.9444444444444444, + "15": 0.9303030303030303, + "16": 0.8290598290598291, + "17": 0.8136645962732919, + "18": 0.7823529411764706, + "19": 0.7053571428571429, + "20": 0.655688622754491 + } + }, + "total_pass_at_k": { + "pass@1": 0.5114481118049361, + "pass@2": 0.5554564377044305, + "pass@4": 0.6027356526910497, + "pass@8": 0.6628010704727921, + "pass@16": 0.7210823669342848, + "pass@32": 0.7766874814154029, + "pass@64": 0.8257508177222718, + "pass@128": 0.8602438299137675 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.601, + "pass@2": 0.674, + "pass@4": 0.763, + "pass@8": 0.854, + "pass@16": 0.927, + "pass@32": 0.963, + "pass@64": 0.979, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.607, + "pass@2": 0.669, + "pass@4": 0.757, + "pass@8": 0.848, + "pass@16": 0.902, + "pass@32": 0.946, + "pass@64": 0.964, + "pass@128": 0.977 + }, + "13": { + "pass@1": 0.598, + "pass@2": 0.648, + "pass@4": 0.713, + "pass@8": 0.795, + "pass@16": 0.855, + "pass@32": 0.906, + "pass@64": 0.946, + "pass@128": 0.965 + }, + "14": { + "pass@1": 0.545, + "pass@2": 0.591, + "pass@4": 0.632, + "pass@8": 0.7, + "pass@16": 0.779, + "pass@32": 0.849, + "pass@64": 0.901, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.513, + "pass@2": 0.55, + "pass@4": 0.589, + "pass@8": 0.647, + "pass@16": 0.716, + "pass@32": 0.804, + "pass@64": 0.875, + "pass@128": 0.922 + }, + "16": { + "pass@1": 0.518, + "pass@2": 0.546, + "pass@4": 0.59, + "pass@8": 0.631, + "pass@16": 0.695, + "pass@32": 0.749, + "pass@64": 0.795, + "pass@128": 0.824 + }, + "17": { + "pass@1": 0.471, + "pass@2": 0.498, + "pass@4": 0.526, + "pass@8": 0.575, + "pass@16": 0.635, + "pass@32": 0.7, + "pass@64": 0.768, + "pass@128": 0.814 + }, + "18": { + "pass@1": 0.45, + "pass@2": 0.498, + "pass@4": 0.522, + "pass@8": 0.554, + "pass@16": 0.601, + "pass@32": 0.654, + "pass@64": 0.721, + "pass@128": 0.792 + }, + "19": { + "pass@1": 0.401, + "pass@2": 0.434, + "pass@4": 0.466, + "pass@8": 0.507, + "pass@16": 0.546, + "pass@32": 0.599, + "pass@64": 0.651, + "pass@128": 0.708 + }, + "20": { + "pass@1": 0.379, + "pass@2": 0.41, + "pass@4": 0.443, + "pass@8": 0.485, + "pass@16": 0.526, + "pass@32": 0.57, + "pass@64": 0.63, + "pass@128": 0.681 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.601, + "12": 0.607, + "13": 0.598, + "14": 0.545, + "15": 0.513, + "16": 0.518, + "17": 0.471, + "18": 0.45, + "19": 0.401, + "20": 0.379 + }, + "pass@2": { + "11": 0.674, + "12": 0.669, + "13": 0.648, + "14": 0.591, + "15": 0.55, + "16": 0.546, + "17": 0.498, + "18": 0.498, + "19": 0.434, + "20": 0.41 + }, + "pass@4": { + "11": 0.763, + "12": 0.757, + "13": 0.713, + "14": 0.632, + "15": 0.589, + "16": 0.59, + "17": 0.526, + "18": 0.522, + "19": 0.466, + "20": 0.443 + }, + "pass@8": { + "11": 0.854, + "12": 0.848, + "13": 0.795, + "14": 0.7, + "15": 0.647, + "16": 0.631, + "17": 0.575, + "18": 0.554, + "19": 0.507, + "20": 0.485 + }, + "pass@16": { + "11": 0.927, + "12": 0.902, + "13": 0.855, + "14": 0.779, + "15": 0.716, + "16": 0.695, + "17": 0.635, + "18": 0.601, + "19": 0.546, + "20": 0.526 + }, + "pass@32": { + "11": 0.963, + "12": 0.946, + "13": 0.906, + "14": 0.849, + "15": 0.804, + "16": 0.749, + "17": 0.7, + "18": 0.654, + "19": 0.599, + "20": 0.57 + }, + "pass@64": { + "11": 0.979, + "12": 0.964, + "13": 0.946, + "14": 0.901, + "15": 0.875, + "16": 0.795, + "17": 0.768, + "18": 0.721, + "19": 0.651, + "20": 0.63 + }, + "pass@128": { + "11": 0.986, + "12": 0.977, + "13": 0.965, + "14": 0.937, + "15": 0.922, + "16": 0.824, + "17": 0.814, + "18": 0.792, + "19": 0.708, + "20": 0.681 + } + }, + "total_pass_at_k": { + "pass@1": 0.5083, + "pass@2": 0.5518, + "pass@4": 0.6001, + "pass@8": 0.6596, + "pass@16": 0.7182, + "pass@32": 0.774, + "pass@64": 0.823, + "pass@128": 0.8606 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/summary.csv b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..7b63b21764d5c9df17980c5568a936d7b661a2fc --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-39470,0.9900894097222223,0.87708359375,0.9306126644736842,0.0,0.0,0.0,200.35493229166667,383.22090859375,1152000,1280000,2432000 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/summary.json b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b7d362f704fca8df90586f47c2d29fe11dac6712 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-39470", + "id_acc": 0.9900894097222223, + "ood_acc": 0.87708359375, + "total_acc": 0.9306126644736842, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.35493229166667, + "ood_avg_resp_len": 383.22090859375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 751583, + "answer_accuracy": 0.9281919360575404, + "avg_response_len": 283.65910527979764, + "resp_tokens_sum": 229686720, + "pass_at_k": { + "pass@1": 0.9281919360575404, + "pass@2": 0.9499665173848083, + "pass@4": 0.9640744475635182, + "pass@8": 0.9737280900028716, + "pass@16": 0.980602757648027, + "pass@32": 0.9857206319262101, + "pass@64": 0.9895329614718181, + "pass@128": 0.9920961112867531 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392514749262537, + "pass@2": 0.9528702268134623, + "pass@4": 0.9647196697204883, + "pass@8": 0.9752039016665607, + "pass@16": 0.9841804774412467, + "pass@32": 0.9916424699073122, + "pass@64": 0.9975593656716509, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9964162844036697, + "pass@2": 0.9992663259409089, + "pass@4": 0.9999513056854131, + "pass@8": 0.9999997214289341, + "pass@16": 0.9999999999945326, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99814453125, + "pass@2": 0.9990688053641733, + "pass@4": 0.9997314116985375, + "pass@8": 0.9999793655718723, + "pass@16": 0.9999999169698948, + "pass@32": 0.9999999999998159, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982886904761905, + "pass@2": 0.9999117297837768, + "pass@4": 0.9999994979794193, + "pass@8": 0.9999999999856092, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9999620591494178, + "pass@4": 0.9999997176625378, + "pass@8": 0.9999999999937114, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9974112426035503, + "pass@2": 0.9990299439733494, + "pass@4": 0.999743682649876, + "pass@8": 0.9999804637850213, + "pass@16": 0.9999999213916163, + "pass@32": 0.9999999999998258, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9911608573717948, + "pass@2": 0.9963473052947709, + "pass@4": 0.9984855673088942, + "pass@8": 0.9994635469329705, + "pass@16": 0.9999228935849399, + "pass@32": 0.9999990402568896, + "pass@64": 0.9999999999948799, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9897447183098591, + "pass@2": 0.9935465648220028, + "pass@4": 0.9950367868629096, + "pass@8": 0.9961759013482411, + "pass@16": 0.9975146715824619, + "pass@32": 0.9990370240116533, + "pass@64": 0.9999221200063837, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9560685198300283, + "pass@2": 0.972025147219558, + "pass@4": 0.9833676431380923, + "pass@8": 0.9902832202553371, + "pass@16": 0.9946957855807873, + "pass@32": 0.997549645536328, + "pass@64": 0.9992147611382037, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.943215925141243, + "pass@2": 0.9660565138573776, + "pass@4": 0.9814934266691235, + "pass@8": 0.9904757233909742, + "pass@16": 0.9957168814771249, + "pass@32": 0.9984134645336938, + "pass@64": 0.9996455904087194, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9108787225705329, + "pass@2": 0.9434656776837063, + "pass@4": 0.9667659061230197, + "pass@8": 0.9808162550485344, + "pass@16": 0.9881944688814267, + "pass@32": 0.9930606223263609, + "pass@64": 0.9966675055170503, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9186380451713395, + "pass@2": 0.9460888249074009, + "pass@4": 0.9653372817299704, + "pass@8": 0.9779994019698924, + "pass@16": 0.9860626693919856, + "pass@32": 0.9919049064530143, + "pass@64": 0.9963014598578678, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9521540823699421, + "pass@2": 0.9741484513677119, + "pass@4": 0.985112245633313, + "pass@8": 0.9908469302823427, + "pass@16": 0.9934400862091718, + "pass@32": 0.994992084796452, + "pass@64": 0.9963530196364373, + "pass@128": 0.9971098265895953 + }, + "16": { + "pass@1": 0.8628276209677419, + "pass@2": 0.9018546005842007, + "pass@4": 0.9287450191709906, + "pass@8": 0.9468683936291992, + "pass@16": 0.9613419346632679, + "pass@32": 0.9736111775442221, + "pass@64": 0.9823099408096163, + "pass@128": 0.9870967741935484 + }, + "17": { + "pass@1": 0.884709176300578, + "pass@2": 0.9211279214646582, + "pass@4": 0.9415992674867953, + "pass@8": 0.9565759689885837, + "pass@16": 0.9692118557186207, + "pass@32": 0.9795491000031725, + "pass@64": 0.98639472632689, + "pass@128": 0.9884393063583815 + }, + "18": { + "pass@1": 0.82296875, + "pass@2": 0.8701905230596182, + "pass@4": 0.8998161420001068, + "pass@8": 0.9205166862174053, + "pass@16": 0.9355630508173823, + "pass@32": 0.9477589149914283, + "pass@64": 0.9590373492455058, + "pass@128": 0.9685714285714285 + }, + "19": { + "pass@1": 0.7173188813813813, + "pass@2": 0.7844284983802695, + "pass@4": 0.8315280263390493, + "pass@8": 0.8687886876667537, + "pass@16": 0.8990525439832475, + "pass@32": 0.9216565068026786, + "pass@64": 0.9364336919565242, + "pass@128": 0.9459459459459459 + }, + "20": { + "pass@1": 0.7638419669669669, + "pass@2": 0.8345680818613895, + "pass@4": 0.8795241003095331, + "pass@8": 0.9096289064963395, + "pass@16": 0.9287254882088332, + "pass@32": 0.9412760341681837, + "pass@64": 0.9527062784829503, + "pass@128": 0.963963963963964 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 766904, + "answer_accuracy": 0.933973109898675, + "avg_response_len": 292.6585639127046, + "resp_tokens_sum": 240307800, + "pass_at_k": { + "pass@1": 0.933973109898675, + "pass@2": 0.9541137397892366, + "pass@4": 0.9676575266507484, + "pass@8": 0.9769106752250937, + "pass@16": 0.9831209304949401, + "pass@32": 0.9874264895458542, + "pass@64": 0.9906284085148328, + "pass@128": 0.9929851909586905 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9263564033742331, + "pass@2": 0.943031511158881, + "pass@4": 0.9574623036959338, + "pass@8": 0.9692292659689711, + "pass@16": 0.9785950061939993, + "pass@32": 0.985816650956142, + "pass@64": 0.9927118784293236, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9957859848484848, + "pass@2": 0.9979222590073968, + "pass@4": 0.9992018259081249, + "pass@8": 0.9998472966011496, + "pass@16": 0.999994268588126, + "pass@32": 0.9999999968257123, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996764053254438, + "pass@2": 0.9999919920328006, + "pass@4": 0.999999990293373, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996286231884058, + "pass@2": 0.9990335786830994, + "pass@4": 0.9998567722512948, + "pass@8": 0.9999954258079576, + "pass@16": 0.9999999965927238, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997089899713467, + "pass@2": 0.9999936545360197, + "pass@4": 0.9999999959711339, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9963872832369942, + "pass@2": 0.9973061080515223, + "pass@4": 0.9977670850211643, + "pass@8": 0.9983013813782712, + "pass@16": 0.999038785005553, + "pass@32": 0.9997319443095561, + "pass@64": 0.9999910524153849, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9921654929577465, + "pass@2": 0.9949418459576357, + "pass@4": 0.9969772593566649, + "pass@8": 0.998742979098145, + "pass@16": 0.9996984788969664, + "pass@32": 0.9999800029461254, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932634730538922, + "pass@2": 0.9957594653213259, + "pass@4": 0.9968023138200539, + "pass@8": 0.9971748134078343, + "pass@16": 0.9973801780816433, + "pass@32": 0.9977544910178684, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9698795180722891, + "pass@2": 0.9801018789725835, + "pass@4": 0.9859459858556837, + "pass@8": 0.9903607486554723, + "pass@16": 0.9944354686155655, + "pass@32": 0.997734424893915, + "pass@64": 0.9996912428943647, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9312793887147336, + "pass@2": 0.9578514921136424, + "pass@4": 0.9785946927088653, + "pass@8": 0.9911220322102386, + "pass@16": 0.9969256254585147, + "pass@32": 0.999356859530079, + "pass@64": 0.9999770727927124, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9410014005602241, + "pass@2": 0.9629413281060455, + "pass@4": 0.9783460898795218, + "pass@8": 0.9885164313887403, + "pass@16": 0.9938448297567265, + "pass@32": 0.99557976291731, + "pass@64": 0.9965040977444787, + "pass@128": 0.9971988795518207 + }, + "14": { + "pass@1": 0.938093932748538, + "pass@2": 0.9617224553345302, + "pass@4": 0.9780836096145873, + "pass@8": 0.9891690673671412, + "pass@16": 0.9953444109429628, + "pass@32": 0.9985866789814201, + "pass@64": 0.9998999204723729, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.946875, + "pass@2": 0.9708445180147938, + "pass@4": 0.9852852896228875, + "pass@8": 0.9932527012437333, + "pass@16": 0.9965859651004563, + "pass@32": 0.9976875706827749, + "pass@64": 0.9984848257468957, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8979255698005698, + "pass@2": 0.9284205560042177, + "pass@4": 0.9476878840999577, + "pass@8": 0.9601126751922175, + "pass@16": 0.9681501715762721, + "pass@32": 0.9745545081734679, + "pass@64": 0.9806737994273104, + "pass@128": 0.9857549857549858 + }, + "17": { + "pass@1": 0.883103649068323, + "pass@2": 0.9193444484520957, + "pass@4": 0.9431839631265031, + "pass@8": 0.9588158770909269, + "pass@16": 0.969734921800441, + "pass@32": 0.9777583996276256, + "pass@64": 0.9834804448754739, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8681985294117647, + "pass@2": 0.9134885942566008, + "pass@4": 0.9388149804803808, + "pass@8": 0.956182315324883, + "pass@16": 0.969199828469146, + "pass@32": 0.9786850953286746, + "pass@64": 0.98495366662752, + "pass@128": 0.9882352941176471 + }, + "19": { + "pass@1": 0.7562779017857143, + "pass@2": 0.8173970929415075, + "pass@4": 0.8586776217704924, + "pass@8": 0.8873359588818277, + "pass@16": 0.9093487259281797, + "pass@32": 0.9286139494295144, + "pass@64": 0.9465647885804772, + "pass@128": 0.9613095238095238 + }, + "20": { + "pass@1": 0.7049962574850299, + "pass@2": 0.7821469511763871, + "pass@4": 0.8387314392587147, + "pass@8": 0.8803005442248173, + "pass@16": 0.9090276962957424, + "pass@32": 0.927911101416634, + "pass@64": 0.9395962270441276, + "pass@128": 0.9461077844311377 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 744763, + "answer_accuracy": 0.9296151042498801, + "avg_response_len": 313.71964995406614, + "resp_tokens_sum": 251337125, + "pass_at_k": { + "pass@1": 0.9296151042498801, + "pass@2": 0.951017935747319, + "pass@4": 0.9651440403059918, + "pass@8": 0.974651918770272, + "pass@16": 0.9810899167866008, + "pass@32": 0.9857287259116158, + "pass@64": 0.9891995583526448, + "pass@128": 0.9915321936411567 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9518889925373134, + "pass@2": 0.9682519978845928, + "pass@4": 0.9806292692891003, + "pass@8": 0.9888008490677124, + "pass@16": 0.9928884788352844, + "pass@32": 0.9939672147924427, + "pass@64": 0.9940297978227394, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999960543835081, + "pass@4": 0.9999999986335526, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992233187134503, + "pass@2": 0.9998215683565871, + "pass@4": 0.9999901437758876, + "pass@8": 0.9999999784883499, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940946691176471, + "pass@2": 0.9972234686197314, + "pass@4": 0.9979508072520346, + "pass@8": 0.9985971409541529, + "pass@16": 0.9993657240569218, + "pass@32": 0.999893411003342, + "pass@64": 0.9999991012534062, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998767744479495, + "pass@2": 0.9999976713281503, + "pass@4": 0.9999999997042957, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9922122231012658, + "pass@2": 0.9943055105651352, + "pass@4": 0.9964102716986326, + "pass@8": 0.9984351822242498, + "pass@16": 0.9996795269701011, + "pass@32": 0.9999902592414398, + "pass@64": 0.9999999992697213, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930555555555556, + "pass@2": 0.9965503150788586, + "pass@4": 0.9984686342698152, + "pass@8": 0.9996429053090278, + "pass@16": 0.9999814246833897, + "pass@32": 0.999999972034264, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9924135852090032, + "pass@2": 0.9953062099653139, + "pass@4": 0.9966570869758643, + "pass@8": 0.9975856061587218, + "pass@16": 0.9983673492925828, + "pass@32": 0.9992571222626693, + "pass@64": 0.9999073347342836, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9599702380952381, + "pass@2": 0.9765876921634795, + "pass@4": 0.989391186815934, + "pass@8": 0.9963180825233957, + "pass@16": 0.9992307852616401, + "pass@32": 0.9999490539138667, + "pass@64": 0.9999999121061375, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.939865252293578, + "pass@2": 0.9603624124106047, + "pass@4": 0.9760637159460566, + "pass@8": 0.9866032202849236, + "pass@16": 0.9926096934140741, + "pass@32": 0.9959497775979793, + "pass@64": 0.9984405444345842, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9073832947530864, + "pass@2": 0.9443962191358023, + "pass@4": 0.9685251580242282, + "pass@8": 0.9828957795842923, + "pass@16": 0.9915318725469653, + "pass@32": 0.996952650279957, + "pass@64": 0.9995923226260632, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9464252596439169, + "pass@2": 0.9695290777821917, + "pass@4": 0.9824109808974172, + "pass@8": 0.9893288765335709, + "pass@16": 0.9930767880716179, + "pass@32": 0.9956401539621311, + "pass@64": 0.996931099741985, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9288194444444444, + "pass@2": 0.9530141197628071, + "pass@4": 0.9679944280691763, + "pass@8": 0.9773730797152483, + "pass@16": 0.9826136664666306, + "pass@32": 0.9854947655239579, + "pass@64": 0.9876295766722463, + "pass@128": 0.9907407407407407 + }, + "16": { + "pass@1": 0.8941049041297935, + "pass@2": 0.932356267275219, + "pass@4": 0.9575642494356341, + "pass@8": 0.9720565681989967, + "pass@16": 0.9788168570235897, + "pass@32": 0.9821212614436396, + "pass@64": 0.9848713086920479, + "pass@128": 0.9882005899705014 + }, + "17": { + "pass@1": 0.8530685240963856, + "pass@2": 0.8834865791907793, + "pass@4": 0.9036101018246212, + "pass@8": 0.9202718219997545, + "pass@16": 0.9349628711134209, + "pass@32": 0.9486017624674481, + "pass@64": 0.9615355997566628, + "pass@128": 0.9728915662650602 + }, + "18": { + "pass@1": 0.8661290322580645, + "pass@2": 0.9098425196850394, + "pass@4": 0.9353601161749935, + "pass@8": 0.9524239545873169, + "pass@16": 0.9650574815675628, + "pass@32": 0.9742872492835252, + "pass@64": 0.9800047314029565, + "pass@128": 0.9838709677419355 + }, + "19": { + "pass@1": 0.7272233761329305, + "pass@2": 0.7992720698432335, + "pass@4": 0.8498617207403449, + "pass@8": 0.886109822147316, + "pass@16": 0.9119255337966973, + "pass@32": 0.9312105063485466, + "pass@64": 0.946458090289178, + "pass@128": 0.9546827794561934 + }, + "20": { + "pass@1": 0.714573948948949, + "pass@2": 0.7877462846704968, + "pass@4": 0.8363218603868204, + "pass@8": 0.8718850949032955, + "pass@16": 0.9008473092143473, + "pass@32": 0.9259147136391661, + "pass@64": 0.945745298634043, + "pass@128": 0.9579579579579579 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_id_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..34cd1ddfb740005f219fdb4546bb42abe997d338 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff0d575672143efef24f9f7b131906ace533f23923fd146e85e9d42bd5d936eb +size 1790134068 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_metrics.json b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f7be8d02416af6236a4d2f24868b37566d4f1dcb --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-39470", + "id": { + "answer_accuracy": 0.9911875, + "avg_response_len": 200.45977864583332, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9911875, + "pass@2": 0.9938752187226583, + "pass@4": 0.9958526534079071, + "pass@8": 0.9974000272396115, + "pass@16": 0.9984365466828654, + "pass@32": 0.9990247440292359, + "pass@64": 0.9993571718356874, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9427734375, + "pass@2": 0.9574158464566929, + "pass@4": 0.9702122222534681, + "pass@8": 0.9808703249072833, + "pass@16": 0.9883692146022091, + "pass@32": 0.9928770993934813, + "pass@64": 0.9954640623735178, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9990859375, + "pass@2": 0.9998460875984252, + "pass@4": 0.9999927702474689, + "pass@8": 0.999999981311028, + "pass@16": 0.9999999999999561, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998984375, + "pass@2": 0.9999954478346457, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951640625, + "pass@2": 0.9975888287401578, + "pass@4": 0.9989402498125234, + "pass@8": 0.9997141524531643, + "pass@16": 0.9999748423354031, + "pass@32": 0.9999998682175585, + "pass@64": 0.9999999999999405, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997421875, + "pass@2": 0.9999659202755905, + "pass@4": 0.9999990039370078, + "pass@8": 0.9999999994855776, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965546875, + "pass@2": 0.9980011072834647, + "pass@4": 0.9987513329583803, + "pass@8": 0.9992684926099065, + "pass@16": 0.9996833288085706, + "pass@32": 0.9999315528119157, + "pass@64": 0.999998555259184, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9942734375, + "pass@2": 0.9968043799212598, + "pass@4": 0.9984263969816272, + "pass@8": 0.9994879949399093, + "pass@16": 0.9999319485291004, + "pass@32": 0.9999988874150888, + "pass@64": 0.9999999999725494, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9931953125, + "pass@2": 0.995259350393701, + "pass@4": 0.9963519162917136, + "pass@8": 0.9972592994496413, + "pass@16": 0.9979695858705498, + "pass@32": 0.9984152884250792, + "pass@64": 0.9987519289159965, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9427734375, + "2": 1.0, + "3": 0.9990859375, + "4": 0.9998984375, + "5": 0.9951640625, + "6": 0.9997421875, + "7": 0.9965546875, + "8": 0.9942734375, + "9": 0.9931953125 + }, + "per_op_avg_response_len": { + "10": 281.8106640625, + "2": 115.647078125, + "3": 145.0691875, + "4": 159.9864765625, + "5": 173.7375546875, + "6": 199.1829609375, + "7": 224.4872265625, + "8": 244.19671875, + "9": 260.020140625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.8106640625, + "2": 115.647078125, + "3": 145.0691875, + "4": 159.9864765625, + "5": 173.7375546875, + "6": 199.1829609375, + "7": 224.4872265625, + "8": 244.19671875, + "9": 260.020140625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375828, + "answer_accuracy": 0.9916096757852076, + "avg_response_len": 190.9041524189463, + "resp_tokens_sum": 72354201, + "pass_at_k": { + "pass@1": 0.9916096757852076, + "pass@2": 0.9939053080466007, + "pass@4": 0.9955002847605042, + "pass@8": 0.9968320723656806, + "pass@16": 0.9979047753381606, + "pass@32": 0.998652277779155, + "pass@64": 0.9991953374623359, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9448285398230089, + "pass@2": 0.9571480210438297, + "pass@4": 0.9679420556390625, + "pass@8": 0.9776938695151067, + "pass@16": 0.9852130050533956, + "pass@32": 0.9901244632055087, + "pass@64": 0.9937076636493979, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.9999759204411375, + "pass@4": 0.9999999051150715, + "pass@8": 0.9999999999996472, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997802734375, + "pass@2": 0.999986158956693, + "pass@4": 0.9999999630905512, + "pass@8": 0.9999999999999802, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984623015873015, + "pass@2": 0.9997957286589176, + "pass@4": 0.9999917819796333, + "pass@8": 0.9999999870037689, + "pass@16": 0.9999999999999949, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9952154215976331, + "pass@2": 0.9968867207519918, + "pass@4": 0.9977492170912068, + "pass@8": 0.9983768112754401, + "pass@16": 0.9991472297701894, + "pass@32": 0.9997987731838602, + "pass@64": 0.999995725631176, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958683894230769, + "pass@2": 0.9983374974762771, + "pass@4": 0.9996146993044138, + "pass@8": 0.9999795440111189, + "pass@16": 0.9999999612415542, + "pass@32": 0.9999999999999838, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9929137323943662, + "pass@2": 0.9947269740490184, + "pass@4": 0.9955706937161022, + "pass@8": 0.9964410384803556, + "pass@16": 0.9974565619535367, + "pass@32": 0.9983808905387922, + "pass@64": 0.9993012422126566, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387053, + "answer_accuracy": 0.9907770519331586, + "avg_response_len": 198.43381133273263, + "resp_tokens_sum": 77519359, + "pass_at_k": { + "pass@1": 0.9907770519331586, + "pass@2": 0.9937578043570242, + "pass@4": 0.9959418754155568, + "pass@8": 0.9974516400822192, + "pass@16": 0.9983782487837621, + "pass@32": 0.9989244614243098, + "pass@64": 0.9992334692907294, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.93498370398773, + "pass@2": 0.9526977863388246, + "pass@4": 0.968151575953313, + "pass@8": 0.980014577561254, + "pass@16": 0.9879659045300164, + "pass@32": 0.9929990063954127, + "pass@64": 0.995891252380096, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977272727272727, + "pass@2": 0.9995574594368886, + "pass@4": 0.9999781856813353, + "pass@8": 0.9999999433671007, + "pass@16": 0.9999999999998672, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9969655797101449, + "pass@2": 0.99905782836928, + "pass@4": 0.9998641240497111, + "pass@8": 0.9999974180784186, + "pass@16": 0.9999999995544241, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999955229226361, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981259031791907, + "pass@2": 0.9992767454826817, + "pass@4": 0.9998384843332444, + "pass@8": 0.9999921708624612, + "pass@16": 0.9999999886553311, + "pass@32": 0.9999999999999982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9939700704225352, + "pass@2": 0.9960279887989353, + "pass@4": 0.9978994763858744, + "pass@8": 0.9993226389661074, + "pass@16": 0.9999261001036863, + "pass@32": 0.9999993808196772, + "pass@64": 0.999999999998295, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930061751497006, + "pass@2": 0.9955111921825641, + "pass@4": 0.9965653779617369, + "pass@8": 0.9969513016358855, + "pass@16": 0.99700529626788, + "pass@32": 0.9970059879973182, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378967, + "answer_accuracy": 0.9911883788081687, + "avg_response_len": 212.0022833319384, + "resp_tokens_sum": 81056105, + "pass_at_k": { + "pass@1": 0.9911883788081687, + "pass@2": 0.9939653607232393, + "pass@4": 0.9961107913381432, + "pass@8": 0.9979103024274498, + "pass@16": 0.9990232558022942, + "pass@32": 0.9994964330405262, + "pass@64": 0.9996439919718464, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9482742537313433, + "pass@2": 0.9622781760488895, + "pass@4": 0.9745147809881973, + "pass@8": 0.984917465243863, + "pass@16": 0.9919555851114118, + "pass@32": 0.9955439709907747, + "pass@64": 0.9968257197625695, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99995444606414, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 0.9999996402587834, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9902803308823529, + "pass@2": 0.9940535983094023, + "pass@4": 0.9970285755640838, + "pass@8": 0.9991619038527141, + "pass@16": 0.9999260073209658, + "pass@32": 0.9999996124045837, + "pass@64": 0.9999999999998251, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992852917981072, + "pass@2": 0.9998924929829356, + "pass@4": 0.9999968578454508, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9962668117088608, + "pass@2": 0.9977963345958338, + "pass@4": 0.9986328544374992, + "pass@8": 0.999429871140494, + "pass@16": 0.9999100287072848, + "pass@32": 0.9999986312524417, + "pass@64": 0.9999999999868562, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931024774774775, + "pass@2": 0.9961956296848028, + "pass@4": 0.9978747648098041, + "pass@8": 0.9992037248512674, + "pass@16": 0.9998744597144952, + "pass@32": 0.9999973189913167, + "pass@64": 0.9999999999193833, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9937198553054662, + "pass@2": 0.9955965801200093, + "pass@4": 0.9970144173095726, + "pass@8": 0.998524103031284, + "pass@16": 0.9995907955741878, + "pass@32": 0.9999680781115876, + "pass@64": 0.9999999611913292, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.88186015625, + "avg_response_len": 382.9250015625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.88186015625, + "pass@2": 0.917877915846434, + "pass@4": 0.941898623884509, + "pass@8": 0.9584196328306133, + "pass@16": 0.9696874531505953, + "pass@32": 0.9776518155520442, + "pass@64": 0.9835516456632658, + "pass@128": 0.988 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9677578125, + "pass@2": 0.9796870078740157, + "pass@4": 0.9879049421634793, + "pass@8": 0.9930438475318435, + "pass@16": 0.9962345896043963, + "pass@32": 0.998368927016233, + "pass@64": 0.9996713476088036, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9471796875, + "pass@2": 0.9683945620078735, + "pass@4": 0.9829806270153733, + "pass@8": 0.9913171642257499, + "pass@16": 0.9955083716733185, + "pass@32": 0.9975559830601772, + "pass@64": 0.9985963318615899, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.9221015625, + "pass@2": 0.9504410679133856, + "pass@4": 0.9713373732658422, + "pass@8": 0.9851710218413028, + "pass@16": 0.9932148515789906, + "pass@32": 0.99749780169572, + "pass@64": 0.999297449000056, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.93978125, + "pass@2": 0.961339197834646, + "pass@4": 0.9756711212973378, + "pass@8": 0.9855161409851468, + "pass@16": 0.9920009967652241, + "pass@32": 0.9962494297192892, + "pass@64": 0.9985943987435743, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9493828125, + "pass@2": 0.9712054625984258, + "pass@4": 0.9833473552680915, + "pass@8": 0.989414792976728, + "pass@16": 0.9919192779624594, + "pass@32": 0.9936364650840056, + "pass@64": 0.9953827064172065, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.8892109375, + "pass@2": 0.9223884104330712, + "pass@4": 0.9458036232658414, + "pass@8": 0.9624387302898952, + "pass@16": 0.9736878675771208, + "pass@32": 0.9818827057572707, + "pass@64": 0.9887627214080997, + "pass@128": 0.995 + }, + "17": { + "pass@1": 0.8745625, + "pass@2": 0.90911626476378, + "pass@4": 0.9327734442257218, + "pass@8": 0.9497935055846174, + "pass@16": 0.9616636017876808, + "pass@32": 0.9700921059221204, + "pass@64": 0.9762516668265676, + "pass@128": 0.981 + }, + "18": { + "pass@1": 0.86415625, + "pass@2": 0.9077465551181106, + "pass@4": 0.9344479491001128, + "pass@8": 0.9533949268281803, + "pass@16": 0.9670471082950732, + "pass@32": 0.9768218470525603, + "pass@64": 0.9840441971970608, + "pass@128": 0.99 + }, + "19": { + "pass@1": 0.7359609375, + "pass@2": 0.8015488435039378, + "pass@4": 0.8483440685226838, + "pass@8": 0.8844303981279618, + "pass@16": 0.9125773973872153, + "pass@32": 0.9341005005504791, + "pass@64": 0.9504253722383746, + "pass@128": 0.962 + }, + "20": { + "pass@1": 0.7285078125, + "pass@2": 0.8069117864173238, + "pass@4": 0.8563757347206595, + "pass@8": 0.8896757999147098, + "pass@16": 0.9130204688744356, + "pass@32": 0.9303123896625576, + "pass@64": 0.9444902653313104, + "pass@128": 0.956 + } + }, + "per_op_accuracy": { + "11": 0.9677578125, + "12": 0.9471796875, + "13": 0.9221015625, + "14": 0.93978125, + "15": 0.9493828125, + "16": 0.8892109375, + "17": 0.8745625, + "18": 0.86415625, + "19": 0.7359609375, + "20": 0.7285078125 + }, + "per_op_avg_response_len": { + "11": 301.2928046875, + "12": 313.564875, + "13": 334.422875, + "14": 361.902546875, + "15": 381.7577109375, + "16": 393.5160546875, + "17": 414.422328125, + "18": 432.7089140625, + "19": 436.769359375, + "20": 458.892546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 301.2928046875, + "12": 313.564875, + "13": 334.422875, + "14": 361.902546875, + "15": 381.7577109375, + "16": 393.5160546875, + "17": 414.422328125, + "18": 432.7089140625, + "19": 436.769359375, + "20": 458.892546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 379632, + "answer_accuracy": 0.8813893016344725, + "avg_response_len": 364.86216799777117, + "resp_tokens_sum": 157153433, + "pass_at_k": { + "pass@1": 0.8813893016344725, + "pass@2": 0.9164895841864537, + "pass@4": 0.9399737569300878, + "pass@8": 0.9565235252060313, + "pass@16": 0.9682947187725689, + "pass@32": 0.9767211107386491, + "pass@64": 0.982554991201546, + "pass@128": 0.986329866270431 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9613137393767706, + "pass@2": 0.9761866091543798, + "pass@4": 0.9861841229655075, + "pass@8": 0.9917381596045705, + "pass@16": 0.9950505009622669, + "pass@32": 0.9975656996714903, + "pass@64": 0.9994669160637162, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9483139124293786, + "pass@2": 0.9684827366653324, + "pass@4": 0.9814987093774294, + "pass@8": 0.988465124155781, + "pass@16": 0.9923368068718912, + "pass@32": 0.9947990848120873, + "pass@64": 0.9963925089910044, + "pass@128": 0.9971751412429378 + }, + "13": { + "pass@1": 0.9207239420062696, + "pass@2": 0.94997323390023, + "pass@4": 0.9702113745382138, + "pass@8": 0.9837949434551917, + "pass@16": 0.9916911106162626, + "pass@32": 0.9957741694909603, + "pass@64": 0.9980404717102529, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9145005841121495, + "pass@2": 0.9406478432310452, + "pass@4": 0.9610959891695777, + "pass@8": 0.976471177377217, + "pass@16": 0.9872585351671593, + "pass@32": 0.9943396934125153, + "pass@64": 0.9980806328920692, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9630825505780347, + "pass@2": 0.980187306563197, + "pass@4": 0.9885191692078952, + "pass@8": 0.9921803061342245, + "pass@16": 0.993480091660752, + "pass@32": 0.9940759267806081, + "pass@64": 0.9942177233079992, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8699596774193549, + "pass@2": 0.908876920878842, + "pass@4": 0.9352617892521493, + "pass@8": 0.9529402191896256, + "pass@16": 0.965859594091936, + "pass@32": 0.9770035584876725, + "pass@64": 0.9864111315573612, + "pass@128": 0.9935483870967742 + }, + "17": { + "pass@1": 0.8939893424855492, + "pass@2": 0.9273410120158392, + "pass@4": 0.9489639912243624, + "pass@8": 0.9627517649968316, + "pass@16": 0.9715418370380917, + "pass@32": 0.9774012892084214, + "pass@64": 0.9808587438167617, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8475223214285714, + "pass@2": 0.8865853487064118, + "pass@4": 0.9117201727462636, + "pass@8": 0.9326498728874107, + "pass@16": 0.949815072080882, + "pass@32": 0.963251620501704, + "pass@64": 0.9739462329256334, + "pass@128": 0.9828571428571429 + }, + "19": { + "pass@1": 0.7332957957957958, + "pass@2": 0.7961523285096122, + "pass@4": 0.8443348072481931, + "pass@8": 0.884565813204154, + "pass@16": 0.9157984814719053, + "pass@32": 0.9363176133739435, + "pass@64": 0.9481651177514631, + "pass@128": 0.954954954954955 + }, + "20": { + "pass@1": 0.7522757132132132, + "pass@2": 0.8239670526826041, + "pass@4": 0.8674565558246655, + "pass@8": 0.8967500518280327, + "pass@16": 0.9183242854815648, + "pass@32": 0.9357294765240374, + "pass@64": 0.949609333521972, + "pass@128": 0.9579579579579579 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 383429, + "answer_accuracy": 0.8907341845078799, + "avg_response_len": 378.4293088388344, + "resp_tokens_sum": 162900194, + "pass_at_k": { + "pass@1": 0.8907341845078799, + "pass@2": 0.9250849038049581, + "pass@4": 0.947908922234412, + "pass@8": 0.9633359839340596, + "pass@16": 0.9733175660470356, + "pass@32": 0.9799818369250699, + "pass@64": 0.9845993355374963, + "pass@128": 0.9881058578650015 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9746799698795181, + "pass@2": 0.9820036790864243, + "pass@4": 0.9869027902950386, + "pass@8": 0.9914568557373679, + "pass@16": 0.9954504095570273, + "pass@32": 0.9981957341656477, + "pass@64": 0.9996185244010403, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9447247257053292, + "pass@2": 0.9670325728778417, + "pass@4": 0.982604836425227, + "pass@8": 0.9920601410768102, + "pass@16": 0.9971194098066611, + "pass@32": 0.999438072623504, + "pass@64": 0.9999856781124098, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9464285714285714, + "pass@2": 0.9673766652330223, + "pass@4": 0.982342608697022, + "pass@8": 0.9916849785119608, + "pass@16": 0.9966756560879082, + "pass@32": 0.9990953691874354, + "pass@64": 0.9999426917920984, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9462262426900585, + "pass@2": 0.9666012657134964, + "pass@4": 0.9802627821193401, + "pass@8": 0.989257075759594, + "pass@16": 0.9947739879169092, + "pass@32": 0.9978547681947219, + "pass@64": 0.9992660885074793, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9564393939393939, + "pass@2": 0.9771869780481988, + "pass@4": 0.9897114525457041, + "pass@8": 0.9962214241568061, + "pass@16": 0.9981651536380237, + "pass@32": 0.9990561561044676, + "pass@64": 0.9998195180147036, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8993723290598291, + "pass@2": 0.9290227443524688, + "pass@4": 0.9515008631399704, + "pass@8": 0.9674285384849971, + "pass@16": 0.9772017098299314, + "pass@32": 0.9839347977212497, + "pass@64": 0.9903181373580099, + "pass@128": 0.9971509971509972 + }, + "17": { + "pass@1": 0.8847777562111802, + "pass@2": 0.9210164541252994, + "pass@4": 0.9453830207365381, + "pass@8": 0.9613599524029054, + "pass@16": 0.9711743000764183, + "pass@32": 0.9769603297072578, + "pass@64": 0.9796747066669252, + "pass@128": 0.9813664596273292 + }, + "18": { + "pass@1": 0.8802619485294118, + "pass@2": 0.9259028340666982, + "pass@4": 0.9532336597631177, + "pass@8": 0.9707902431940827, + "pass@16": 0.9811945823460004, + "pass@32": 0.9869220810987678, + "pass@64": 0.9895681723618133, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7547665550595238, + "pass@2": 0.8166603686257972, + "pass@4": 0.8588360034794753, + "pass@8": 0.8899064276709484, + "pass@16": 0.9129267281645536, + "pass@32": 0.9315558807032726, + "pass@64": 0.9471272503883296, + "pass@128": 0.9613095238095238 + }, + "20": { + "pass@1": 0.718562874251497, + "pass@2": 0.7972754048988645, + "pass@4": 0.8476215479614443, + "pass@8": 0.8824998849887433, + "pass@16": 0.9078157775076584, + "pass@32": 0.9261277125040664, + "pass@64": 0.9398950398697754, + "pass@128": 0.9491017964071856 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 365720, + "answer_accuracy": 0.8732235635696821, + "avg_response_len": 406.1219604790648, + "resp_tokens_sum": 170090375, + "pass_at_k": { + "pass@1": 0.8732235635696821, + "pass@2": 0.911898281228948, + "pass@4": 0.9377007461189172, + "pass@8": 0.955316550127628, + "pass@16": 0.967388700556255, + "pass@32": 0.9762141504602199, + "pass@64": 0.9834998000693228, + "pass@128": 0.9896088019559902 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9676835317460317, + "pass@2": 0.9811679790026249, + "pass@4": 0.9908895885038183, + "pass@8": 0.9961796859892825, + "pass@16": 0.9983880215612163, + "pass@32": 0.9994515913942914, + "pass@64": 0.9999561147211625, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9483467125382263, + "pass@2": 0.9696277751691591, + "pass@4": 0.9849515017618209, + "pass@8": 0.9936798937954158, + "pass@16": 0.9973701844414204, + "pass@32": 0.9987044705498487, + "pass@64": 0.9996267656297112, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.8966531635802469, + "pass@2": 0.9322411629969862, + "pass@4": 0.9603198564299832, + "pass@8": 0.9793484492294058, + "pass@16": 0.9909017841636405, + "pass@32": 0.9974345519388546, + "pass@64": 0.9998240665268706, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9573210311572701, + "pass@2": 0.9757080334820909, + "pass@4": 0.9848944726679192, + "pass@8": 0.9903352318613627, + "pass@16": 0.993704163528141, + "pass@32": 0.9964393394993372, + "pass@64": 0.9984021166636863, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9275655864197531, + "pass@2": 0.9555214560853507, + "pass@4": 0.9713424178459172, + "pass@8": 0.9795288181559892, + "pass@16": 0.983890943047196, + "pass@32": 0.9876471077266684, + "pass@64": 0.9921078185425506, + "pass@128": 0.9969135802469136 + }, + "16": { + "pass@1": 0.8962942477876106, + "pass@2": 0.9278748722504823, + "pass@4": 0.9495447363880404, + "pass@8": 0.965958245819696, + "pass@16": 0.9772082397000456, + "pass@32": 0.9842197304599816, + "pass@64": 0.9893026678839408, + "pass@128": 0.9941002949852508 + }, + "17": { + "pass@1": 0.8444088855421686, + "pass@2": 0.8785812541504603, + "pass@4": 0.9036703933280926, + "pass@8": 0.9250707536806557, + "pass@16": 0.9421445829755859, + "pass@32": 0.9558133544887637, + "pass@64": 0.968130379274754, + "pass@128": 0.9789156626506024 + }, + "18": { + "pass@1": 0.8652721774193548, + "pass@2": 0.9117249015748031, + "pass@4": 0.9395046590950324, + "pass@8": 0.9577380278438656, + "pass@16": 0.9709860486100815, + "pass@32": 0.9810653945270422, + "pass@64": 0.9893865711937819, + "pass@128": 0.9967741935483871 + }, + "19": { + "pass@1": 0.7195524924471299, + "pass@2": 0.7916381699455244, + "pass@4": 0.8417271315405817, + "pass@8": 0.8787354158112974, + "pass@16": 0.9089822428815135, + "pass@32": 0.9344530494883864, + "pass@64": 0.9560472262738914, + "pass@128": 0.9697885196374623 + }, + "20": { + "pass@1": 0.7147147147147147, + "pass@2": 0.7995218397531393, + "pass@4": 0.854075389224995, + "pass@8": 0.8897990122214269, + "pass@16": 0.9129369733378919, + "pass@32": 0.9290925465576422, + "pass@64": 0.9439802220780451, + "pass@128": 0.960960960960961 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9336467927631579, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9336467927631579, + "pass@2": 0.953876638261499, + "pass@4": 0.9674557957640126, + "pass@8": 0.9768840301822466, + "pass@16": 0.9833054448237678, + "pass@32": 0.9877758343043999, + "pass@64": 0.991038473850203, + "pass@128": 0.9934736842105263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9427734375, + "pass@2": 0.9574158464566929, + "pass@4": 0.9702122222534681, + "pass@8": 0.9808703249072833, + "pass@16": 0.9883692146022091, + "pass@32": 0.9928770993934813, + "pass@64": 0.9954640623735178, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9990859375, + "pass@2": 0.9998460875984252, + "pass@4": 0.9999927702474689, + "pass@8": 0.999999981311028, + "pass@16": 0.9999999999999561, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998984375, + "pass@2": 0.9999954478346457, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951640625, + "pass@2": 0.9975888287401578, + "pass@4": 0.9989402498125234, + "pass@8": 0.9997141524531643, + "pass@16": 0.9999748423354031, + "pass@32": 0.9999998682175585, + "pass@64": 0.9999999999999405, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997421875, + "pass@2": 0.9999659202755905, + "pass@4": 0.9999990039370078, + "pass@8": 0.9999999994855776, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965546875, + "pass@2": 0.9980011072834647, + "pass@4": 0.9987513329583803, + "pass@8": 0.9992684926099065, + "pass@16": 0.9996833288085706, + "pass@32": 0.9999315528119157, + "pass@64": 0.999998555259184, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9942734375, + "pass@2": 0.9968043799212598, + "pass@4": 0.9984263969816272, + "pass@8": 0.9994879949399093, + "pass@16": 0.9999319485291004, + "pass@32": 0.9999988874150888, + "pass@64": 0.9999999999725494, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9931953125, + "pass@2": 0.995259350393701, + "pass@4": 0.9963519162917136, + "pass@8": 0.9972592994496413, + "pass@16": 0.9979695858705498, + "pass@32": 0.9984152884250792, + "pass@64": 0.9987519289159965, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9677578125, + "pass@2": 0.9796870078740157, + "pass@4": 0.9879049421634793, + "pass@8": 0.9930438475318435, + "pass@16": 0.9962345896043963, + "pass@32": 0.998368927016233, + "pass@64": 0.9996713476088036, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9471796875, + "pass@2": 0.9683945620078735, + "pass@4": 0.9829806270153733, + "pass@8": 0.9913171642257499, + "pass@16": 0.9955083716733185, + "pass@32": 0.9975559830601772, + "pass@64": 0.9985963318615899, + "pass@128": 0.999 + }, + "13": { + "pass@1": 0.9221015625, + "pass@2": 0.9504410679133856, + "pass@4": 0.9713373732658422, + "pass@8": 0.9851710218413028, + "pass@16": 0.9932148515789906, + "pass@32": 0.99749780169572, + "pass@64": 0.999297449000056, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.93978125, + "pass@2": 0.961339197834646, + "pass@4": 0.9756711212973378, + "pass@8": 0.9855161409851468, + "pass@16": 0.9920009967652241, + "pass@32": 0.9962494297192892, + "pass@64": 0.9985943987435743, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9493828125, + "pass@2": 0.9712054625984258, + "pass@4": 0.9833473552680915, + "pass@8": 0.989414792976728, + "pass@16": 0.9919192779624594, + "pass@32": 0.9936364650840056, + "pass@64": 0.9953827064172065, + "pass@128": 0.997 + }, + "16": { + "pass@1": 0.8892109375, + "pass@2": 0.9223884104330712, + "pass@4": 0.9458036232658414, + "pass@8": 0.9624387302898952, + "pass@16": 0.9736878675771208, + "pass@32": 0.9818827057572707, + "pass@64": 0.9887627214080997, + "pass@128": 0.995 + }, + "17": { + "pass@1": 0.8745625, + "pass@2": 0.90911626476378, + "pass@4": 0.9327734442257218, + "pass@8": 0.9497935055846174, + "pass@16": 0.9616636017876808, + "pass@32": 0.9700921059221204, + "pass@64": 0.9762516668265676, + "pass@128": 0.981 + }, + "18": { + "pass@1": 0.86415625, + "pass@2": 0.9077465551181106, + "pass@4": 0.9344479491001128, + "pass@8": 0.9533949268281803, + "pass@16": 0.9670471082950732, + "pass@32": 0.9768218470525603, + "pass@64": 0.9840441971970608, + "pass@128": 0.99 + }, + "19": { + "pass@1": 0.7359609375, + "pass@2": 0.8015488435039378, + "pass@4": 0.8483440685226838, + "pass@8": 0.8844303981279618, + "pass@16": 0.9125773973872153, + "pass@32": 0.9341005005504791, + "pass@64": 0.9504253722383746, + "pass@128": 0.962 + }, + "20": { + "pass@1": 0.7285078125, + "pass@2": 0.8069117864173238, + "pass@4": 0.8563757347206595, + "pass@8": 0.8896757999147098, + "pass@16": 0.9130204688744356, + "pass@32": 0.9303123896625576, + "pass@64": 0.9444902653313104, + "pass@128": 0.956 + } + }, + "per_op_accuracy": { + "10": 0.9427734375, + "2": 1.0, + "3": 0.9990859375, + "4": 0.9998984375, + "5": 0.9951640625, + "6": 0.9997421875, + "7": 0.9965546875, + "8": 0.9942734375, + "9": 0.9931953125, + "11": 0.9677578125, + "12": 0.9471796875, + "13": 0.9221015625, + "14": 0.93978125, + "15": 0.9493828125, + "16": 0.8892109375, + "17": 0.8745625, + "18": 0.86415625, + "19": 0.7359609375, + "20": 0.7285078125 + }, + "per_op_avg_response_len": { + "10": 281.8106640625, + "2": 115.647078125, + "3": 145.0691875, + "4": 159.9864765625, + "5": 173.7375546875, + "6": 199.1829609375, + "7": 224.4872265625, + "8": 244.19671875, + "9": 260.020140625, + "11": 301.2928046875, + "12": 313.564875, + "13": 334.422875, + "14": 361.902546875, + "15": 381.7577109375, + "16": 393.5160546875, + "17": 414.422328125, + "18": 432.7089140625, + "19": 436.769359375, + "20": 458.892546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.8106640625, + "2": 115.647078125, + "3": 145.0691875, + "4": 159.9864765625, + "5": 173.7375546875, + "6": 199.1829609375, + "7": 224.4872265625, + "8": 244.19671875, + "9": 260.020140625, + "11": 301.2928046875, + "12": 313.564875, + "13": 334.422875, + "14": 361.902546875, + "15": 381.7577109375, + "16": 393.5160546875, + "17": 414.422328125, + "18": 432.7089140625, + "19": 436.769359375, + "20": 458.892546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 755460, + "answer_accuracy": 0.9329799636421119, + "avg_response_len": 283.43793718384444, + "resp_tokens_sum": 229507634, + "pass_at_k": { + "pass@1": 0.9329799636421119, + "pass@2": 0.952725429641686, + "pass@4": 0.9659639638390104, + "pass@8": 0.9753906779312447, + "pass@16": 0.9821542473041406, + "pass@32": 0.9869863945841979, + "pass@64": 0.9903438096141625, + "pass@128": 0.9925703446095478 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9448285398230089, + "pass@2": 0.9571480210438297, + "pass@4": 0.9679420556390625, + "pass@8": 0.9776938695151067, + "pass@16": 0.9852130050533956, + "pass@32": 0.9901244632055087, + "pass@64": 0.9937076636493979, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.9999759204411375, + "pass@4": 0.9999999051150715, + "pass@8": 0.9999999999996472, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997802734375, + "pass@2": 0.999986158956693, + "pass@4": 0.9999999630905512, + "pass@8": 0.9999999999999802, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984623015873015, + "pass@2": 0.9997957286589176, + "pass@4": 0.9999917819796333, + "pass@8": 0.9999999870037689, + "pass@16": 0.9999999999999949, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9952154215976331, + "pass@2": 0.9968867207519918, + "pass@4": 0.9977492170912068, + "pass@8": 0.9983768112754401, + "pass@16": 0.9991472297701894, + "pass@32": 0.9997987731838602, + "pass@64": 0.999995725631176, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958683894230769, + "pass@2": 0.9983374974762771, + "pass@4": 0.9996146993044138, + "pass@8": 0.9999795440111189, + "pass@16": 0.9999999612415542, + "pass@32": 0.9999999999999838, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9929137323943662, + "pass@2": 0.9947269740490184, + "pass@4": 0.9955706937161022, + "pass@8": 0.9964410384803556, + "pass@16": 0.9974565619535367, + "pass@32": 0.9983808905387922, + "pass@64": 0.9993012422126566, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9613137393767706, + "pass@2": 0.9761866091543798, + "pass@4": 0.9861841229655075, + "pass@8": 0.9917381596045705, + "pass@16": 0.9950505009622669, + "pass@32": 0.9975656996714903, + "pass@64": 0.9994669160637162, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9483139124293786, + "pass@2": 0.9684827366653324, + "pass@4": 0.9814987093774294, + "pass@8": 0.988465124155781, + "pass@16": 0.9923368068718912, + "pass@32": 0.9947990848120873, + "pass@64": 0.9963925089910044, + "pass@128": 0.9971751412429378 + }, + "13": { + "pass@1": 0.9207239420062696, + "pass@2": 0.94997323390023, + "pass@4": 0.9702113745382138, + "pass@8": 0.9837949434551917, + "pass@16": 0.9916911106162626, + "pass@32": 0.9957741694909603, + "pass@64": 0.9980404717102529, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9145005841121495, + "pass@2": 0.9406478432310452, + "pass@4": 0.9610959891695777, + "pass@8": 0.976471177377217, + "pass@16": 0.9872585351671593, + "pass@32": 0.9943396934125153, + "pass@64": 0.9980806328920692, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9630825505780347, + "pass@2": 0.980187306563197, + "pass@4": 0.9885191692078952, + "pass@8": 0.9921803061342245, + "pass@16": 0.993480091660752, + "pass@32": 0.9940759267806081, + "pass@64": 0.9942177233079992, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8699596774193549, + "pass@2": 0.908876920878842, + "pass@4": 0.9352617892521493, + "pass@8": 0.9529402191896256, + "pass@16": 0.965859594091936, + "pass@32": 0.9770035584876725, + "pass@64": 0.9864111315573612, + "pass@128": 0.9935483870967742 + }, + "17": { + "pass@1": 0.8939893424855492, + "pass@2": 0.9273410120158392, + "pass@4": 0.9489639912243624, + "pass@8": 0.9627517649968316, + "pass@16": 0.9715418370380917, + "pass@32": 0.9774012892084214, + "pass@64": 0.9808587438167617, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8475223214285714, + "pass@2": 0.8865853487064118, + "pass@4": 0.9117201727462636, + "pass@8": 0.9326498728874107, + "pass@16": 0.949815072080882, + "pass@32": 0.963251620501704, + "pass@64": 0.9739462329256334, + "pass@128": 0.9828571428571429 + }, + "19": { + "pass@1": 0.7332957957957958, + "pass@2": 0.7961523285096122, + "pass@4": 0.8443348072481931, + "pass@8": 0.884565813204154, + "pass@16": 0.9157984814719053, + "pass@32": 0.9363176133739435, + "pass@64": 0.9481651177514631, + "pass@128": 0.954954954954955 + }, + "20": { + "pass@1": 0.7522757132132132, + "pass@2": 0.8239670526826041, + "pass@4": 0.8674565558246655, + "pass@8": 0.8967500518280327, + "pass@16": 0.9183242854815648, + "pass@32": 0.9357294765240374, + "pass@64": 0.949609333521972, + "pass@128": 0.9579579579579579 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 770482, + "answer_accuracy": 0.9383305728760717, + "avg_response_len": 292.7946621687451, + "resp_tokens_sum": 240419553, + "pass_at_k": { + "pass@1": 0.9383305728760717, + "pass@2": 0.9577567186895759, + "pass@4": 0.9707610770448315, + "pass@8": 0.9795668463758612, + "pass@16": 0.9852404349032309, + "pass@32": 0.9889939787756831, + "pass@64": 0.9915616700994422, + "pass@128": 0.9934528448947778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.93498370398773, + "pass@2": 0.9526977863388246, + "pass@4": 0.968151575953313, + "pass@8": 0.980014577561254, + "pass@16": 0.9879659045300164, + "pass@32": 0.9929990063954127, + "pass@64": 0.995891252380096, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977272727272727, + "pass@2": 0.9995574594368886, + "pass@4": 0.9999781856813353, + "pass@8": 0.9999999433671007, + "pass@16": 0.9999999999998672, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9969655797101449, + "pass@2": 0.99905782836928, + "pass@4": 0.9998641240497111, + "pass@8": 0.9999974180784186, + "pass@16": 0.9999999995544241, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999955229226361, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981259031791907, + "pass@2": 0.9992767454826817, + "pass@4": 0.9998384843332444, + "pass@8": 0.9999921708624612, + "pass@16": 0.9999999886553311, + "pass@32": 0.9999999999999982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9939700704225352, + "pass@2": 0.9960279887989353, + "pass@4": 0.9978994763858744, + "pass@8": 0.9993226389661074, + "pass@16": 0.9999261001036863, + "pass@32": 0.9999993808196772, + "pass@64": 0.999999999998295, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930061751497006, + "pass@2": 0.9955111921825641, + "pass@4": 0.9965653779617369, + "pass@8": 0.9969513016358855, + "pass@16": 0.99700529626788, + "pass@32": 0.9970059879973182, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9746799698795181, + "pass@2": 0.9820036790864243, + "pass@4": 0.9869027902950386, + "pass@8": 0.9914568557373679, + "pass@16": 0.9954504095570273, + "pass@32": 0.9981957341656477, + "pass@64": 0.9996185244010403, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9447247257053292, + "pass@2": 0.9670325728778417, + "pass@4": 0.982604836425227, + "pass@8": 0.9920601410768102, + "pass@16": 0.9971194098066611, + "pass@32": 0.999438072623504, + "pass@64": 0.9999856781124098, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9464285714285714, + "pass@2": 0.9673766652330223, + "pass@4": 0.982342608697022, + "pass@8": 0.9916849785119608, + "pass@16": 0.9966756560879082, + "pass@32": 0.9990953691874354, + "pass@64": 0.9999426917920984, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9462262426900585, + "pass@2": 0.9666012657134964, + "pass@4": 0.9802627821193401, + "pass@8": 0.989257075759594, + "pass@16": 0.9947739879169092, + "pass@32": 0.9978547681947219, + "pass@64": 0.9992660885074793, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9564393939393939, + "pass@2": 0.9771869780481988, + "pass@4": 0.9897114525457041, + "pass@8": 0.9962214241568061, + "pass@16": 0.9981651536380237, + "pass@32": 0.9990561561044676, + "pass@64": 0.9998195180147036, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8993723290598291, + "pass@2": 0.9290227443524688, + "pass@4": 0.9515008631399704, + "pass@8": 0.9674285384849971, + "pass@16": 0.9772017098299314, + "pass@32": 0.9839347977212497, + "pass@64": 0.9903181373580099, + "pass@128": 0.9971509971509972 + }, + "17": { + "pass@1": 0.8847777562111802, + "pass@2": 0.9210164541252994, + "pass@4": 0.9453830207365381, + "pass@8": 0.9613599524029054, + "pass@16": 0.9711743000764183, + "pass@32": 0.9769603297072578, + "pass@64": 0.9796747066669252, + "pass@128": 0.9813664596273292 + }, + "18": { + "pass@1": 0.8802619485294118, + "pass@2": 0.9259028340666982, + "pass@4": 0.9532336597631177, + "pass@8": 0.9707902431940827, + "pass@16": 0.9811945823460004, + "pass@32": 0.9869220810987678, + "pass@64": 0.9895681723618133, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7547665550595238, + "pass@2": 0.8166603686257972, + "pass@4": 0.8588360034794753, + "pass@8": 0.8899064276709484, + "pass@16": 0.9129267281645536, + "pass@32": 0.9315558807032726, + "pass@64": 0.9471272503883296, + "pass@128": 0.9613095238095238 + }, + "20": { + "pass@1": 0.718562874251497, + "pass@2": 0.7972754048988645, + "pass@4": 0.8476215479614443, + "pass@8": 0.8824998849887433, + "pass@16": 0.9078157775076584, + "pass@32": 0.9261277125040664, + "pass@64": 0.9398950398697754, + "pass@128": 0.9491017964071856 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 744687, + "answer_accuracy": 0.9295202408531714, + "avg_response_len": 313.48168637162485, + "resp_tokens_sum": 251146480, + "pass_at_k": { + "pass@1": 0.9295202408531714, + "pass@2": 0.9510633821155672, + "pass@4": 0.96557593465859, + "pass@8": 0.9756436851523189, + "pass@16": 0.9824857474519137, + "pass@32": 0.9873252190122862, + "pass@64": 0.991204337729149, + "pass@128": 0.9944080524045374 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9482742537313433, + "pass@2": 0.9622781760488895, + "pass@4": 0.9745147809881973, + "pass@8": 0.984917465243863, + "pass@16": 0.9919555851114118, + "pass@32": 0.9955439709907747, + "pass@64": 0.9968257197625695, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99995444606414, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 0.9999996402587834, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9902803308823529, + "pass@2": 0.9940535983094023, + "pass@4": 0.9970285755640838, + "pass@8": 0.9991619038527141, + "pass@16": 0.9999260073209658, + "pass@32": 0.9999996124045837, + "pass@64": 0.9999999999998251, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992852917981072, + "pass@2": 0.9998924929829356, + "pass@4": 0.9999968578454508, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9962668117088608, + "pass@2": 0.9977963345958338, + "pass@4": 0.9986328544374992, + "pass@8": 0.999429871140494, + "pass@16": 0.9999100287072848, + "pass@32": 0.9999986312524417, + "pass@64": 0.9999999999868562, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931024774774775, + "pass@2": 0.9961956296848028, + "pass@4": 0.9978747648098041, + "pass@8": 0.9992037248512674, + "pass@16": 0.9998744597144952, + "pass@32": 0.9999973189913167, + "pass@64": 0.9999999999193833, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9937198553054662, + "pass@2": 0.9955965801200093, + "pass@4": 0.9970144173095726, + "pass@8": 0.998524103031284, + "pass@16": 0.9995907955741878, + "pass@32": 0.9999680781115876, + "pass@64": 0.9999999611913292, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9676835317460317, + "pass@2": 0.9811679790026249, + "pass@4": 0.9908895885038183, + "pass@8": 0.9961796859892825, + "pass@16": 0.9983880215612163, + "pass@32": 0.9994515913942914, + "pass@64": 0.9999561147211625, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9483467125382263, + "pass@2": 0.9696277751691591, + "pass@4": 0.9849515017618209, + "pass@8": 0.9936798937954158, + "pass@16": 0.9973701844414204, + "pass@32": 0.9987044705498487, + "pass@64": 0.9996267656297112, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.8966531635802469, + "pass@2": 0.9322411629969862, + "pass@4": 0.9603198564299832, + "pass@8": 0.9793484492294058, + "pass@16": 0.9909017841636405, + "pass@32": 0.9974345519388546, + "pass@64": 0.9998240665268706, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9573210311572701, + "pass@2": 0.9757080334820909, + "pass@4": 0.9848944726679192, + "pass@8": 0.9903352318613627, + "pass@16": 0.993704163528141, + "pass@32": 0.9964393394993372, + "pass@64": 0.9984021166636863, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9275655864197531, + "pass@2": 0.9555214560853507, + "pass@4": 0.9713424178459172, + "pass@8": 0.9795288181559892, + "pass@16": 0.983890943047196, + "pass@32": 0.9876471077266684, + "pass@64": 0.9921078185425506, + "pass@128": 0.9969135802469136 + }, + "16": { + "pass@1": 0.8962942477876106, + "pass@2": 0.9278748722504823, + "pass@4": 0.9495447363880404, + "pass@8": 0.965958245819696, + "pass@16": 0.9772082397000456, + "pass@32": 0.9842197304599816, + "pass@64": 0.9893026678839408, + "pass@128": 0.9941002949852508 + }, + "17": { + "pass@1": 0.8444088855421686, + "pass@2": 0.8785812541504603, + "pass@4": 0.9036703933280926, + "pass@8": 0.9250707536806557, + "pass@16": 0.9421445829755859, + "pass@32": 0.9558133544887637, + "pass@64": 0.968130379274754, + "pass@128": 0.9789156626506024 + }, + "18": { + "pass@1": 0.8652721774193548, + "pass@2": 0.9117249015748031, + "pass@4": 0.9395046590950324, + "pass@8": 0.9577380278438656, + "pass@16": 0.9709860486100815, + "pass@32": 0.9810653945270422, + "pass@64": 0.9893865711937819, + "pass@128": 0.9967741935483871 + }, + "19": { + "pass@1": 0.7195524924471299, + "pass@2": 0.7916381699455244, + "pass@4": 0.8417271315405817, + "pass@8": 0.8787354158112974, + "pass@16": 0.9089822428815135, + "pass@32": 0.9344530494883864, + "pass@64": 0.9560472262738914, + "pass@128": 0.9697885196374623 + }, + "20": { + "pass@1": 0.7147147147147147, + "pass@2": 0.7995218397531393, + "pass@4": 0.854075389224995, + "pass@8": 0.8897990122214269, + "pass@16": 0.9129369733378919, + "pass@32": 0.9290925465576422, + "pass@64": 0.9439802220780451, + "pass@128": 0.960960960960961 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_ood_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..47a395ddc3b909adc830fb9df69e857028594b1f --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4122a827105f713b27045584f5cfc7fa8196a4e57ad783a78c273c45097e625f +size 2724441314 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/summary.csv b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..339f621da4ebb1bf6685dca178d3cea36c4467df --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-39470,0.9911875,0.88186015625,0.9336467927631579,0.0,0.0,0.0,200.45977864583332,382.9250015625,1152000,1280000,2432000 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/summary.json b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f4f671ffc43a4932e67e53fc003ef33b2c403308 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-39470", + "id_acc": 0.9911875, + "ood_acc": 0.88186015625, + "total_acc": 0.9336467927631579, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.45977864583332, + "ood_avg_resp_len": 382.9250015625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 755460, + "answer_accuracy": 0.9329799636421119, + "avg_response_len": 283.43793718384444, + "resp_tokens_sum": 229507634, + "pass_at_k": { + "pass@1": 0.9329799636421119, + "pass@2": 0.952725429641686, + "pass@4": 0.9659639638390104, + "pass@8": 0.9753906779312447, + "pass@16": 0.9821542473041406, + "pass@32": 0.9869863945841979, + "pass@64": 0.9903438096141625, + "pass@128": 0.9925703446095478 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9448285398230089, + "pass@2": 0.9571480210438297, + "pass@4": 0.9679420556390625, + "pass@8": 0.9776938695151067, + "pass@16": 0.9852130050533956, + "pass@32": 0.9901244632055087, + "pass@64": 0.9937076636493979, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.9999759204411375, + "pass@4": 0.9999999051150715, + "pass@8": 0.9999999999996472, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997802734375, + "pass@2": 0.999986158956693, + "pass@4": 0.9999999630905512, + "pass@8": 0.9999999999999802, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984623015873015, + "pass@2": 0.9997957286589176, + "pass@4": 0.9999917819796333, + "pass@8": 0.9999999870037689, + "pass@16": 0.9999999999999949, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9952154215976331, + "pass@2": 0.9968867207519918, + "pass@4": 0.9977492170912068, + "pass@8": 0.9983768112754401, + "pass@16": 0.9991472297701894, + "pass@32": 0.9997987731838602, + "pass@64": 0.999995725631176, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958683894230769, + "pass@2": 0.9983374974762771, + "pass@4": 0.9996146993044138, + "pass@8": 0.9999795440111189, + "pass@16": 0.9999999612415542, + "pass@32": 0.9999999999999838, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9929137323943662, + "pass@2": 0.9947269740490184, + "pass@4": 0.9955706937161022, + "pass@8": 0.9964410384803556, + "pass@16": 0.9974565619535367, + "pass@32": 0.9983808905387922, + "pass@64": 0.9993012422126566, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9613137393767706, + "pass@2": 0.9761866091543798, + "pass@4": 0.9861841229655075, + "pass@8": 0.9917381596045705, + "pass@16": 0.9950505009622669, + "pass@32": 0.9975656996714903, + "pass@64": 0.9994669160637162, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9483139124293786, + "pass@2": 0.9684827366653324, + "pass@4": 0.9814987093774294, + "pass@8": 0.988465124155781, + "pass@16": 0.9923368068718912, + "pass@32": 0.9947990848120873, + "pass@64": 0.9963925089910044, + "pass@128": 0.9971751412429378 + }, + "13": { + "pass@1": 0.9207239420062696, + "pass@2": 0.94997323390023, + "pass@4": 0.9702113745382138, + "pass@8": 0.9837949434551917, + "pass@16": 0.9916911106162626, + "pass@32": 0.9957741694909603, + "pass@64": 0.9980404717102529, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9145005841121495, + "pass@2": 0.9406478432310452, + "pass@4": 0.9610959891695777, + "pass@8": 0.976471177377217, + "pass@16": 0.9872585351671593, + "pass@32": 0.9943396934125153, + "pass@64": 0.9980806328920692, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9630825505780347, + "pass@2": 0.980187306563197, + "pass@4": 0.9885191692078952, + "pass@8": 0.9921803061342245, + "pass@16": 0.993480091660752, + "pass@32": 0.9940759267806081, + "pass@64": 0.9942177233079992, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8699596774193549, + "pass@2": 0.908876920878842, + "pass@4": 0.9352617892521493, + "pass@8": 0.9529402191896256, + "pass@16": 0.965859594091936, + "pass@32": 0.9770035584876725, + "pass@64": 0.9864111315573612, + "pass@128": 0.9935483870967742 + }, + "17": { + "pass@1": 0.8939893424855492, + "pass@2": 0.9273410120158392, + "pass@4": 0.9489639912243624, + "pass@8": 0.9627517649968316, + "pass@16": 0.9715418370380917, + "pass@32": 0.9774012892084214, + "pass@64": 0.9808587438167617, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8475223214285714, + "pass@2": 0.8865853487064118, + "pass@4": 0.9117201727462636, + "pass@8": 0.9326498728874107, + "pass@16": 0.949815072080882, + "pass@32": 0.963251620501704, + "pass@64": 0.9739462329256334, + "pass@128": 0.9828571428571429 + }, + "19": { + "pass@1": 0.7332957957957958, + "pass@2": 0.7961523285096122, + "pass@4": 0.8443348072481931, + "pass@8": 0.884565813204154, + "pass@16": 0.9157984814719053, + "pass@32": 0.9363176133739435, + "pass@64": 0.9481651177514631, + "pass@128": 0.954954954954955 + }, + "20": { + "pass@1": 0.7522757132132132, + "pass@2": 0.8239670526826041, + "pass@4": 0.8674565558246655, + "pass@8": 0.8967500518280327, + "pass@16": 0.9183242854815648, + "pass@32": 0.9357294765240374, + "pass@64": 0.949609333521972, + "pass@128": 0.9579579579579579 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 770482, + "answer_accuracy": 0.9383305728760717, + "avg_response_len": 292.7946621687451, + "resp_tokens_sum": 240419553, + "pass_at_k": { + "pass@1": 0.9383305728760717, + "pass@2": 0.9577567186895759, + "pass@4": 0.9707610770448315, + "pass@8": 0.9795668463758612, + "pass@16": 0.9852404349032309, + "pass@32": 0.9889939787756831, + "pass@64": 0.9915616700994422, + "pass@128": 0.9934528448947778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.93498370398773, + "pass@2": 0.9526977863388246, + "pass@4": 0.968151575953313, + "pass@8": 0.980014577561254, + "pass@16": 0.9879659045300164, + "pass@32": 0.9929990063954127, + "pass@64": 0.995891252380096, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977272727272727, + "pass@2": 0.9995574594368886, + "pass@4": 0.9999781856813353, + "pass@8": 0.9999999433671007, + "pass@16": 0.9999999999998672, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9969655797101449, + "pass@2": 0.99905782836928, + "pass@4": 0.9998641240497111, + "pass@8": 0.9999974180784186, + "pass@16": 0.9999999995544241, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999955229226361, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981259031791907, + "pass@2": 0.9992767454826817, + "pass@4": 0.9998384843332444, + "pass@8": 0.9999921708624612, + "pass@16": 0.9999999886553311, + "pass@32": 0.9999999999999982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9939700704225352, + "pass@2": 0.9960279887989353, + "pass@4": 0.9978994763858744, + "pass@8": 0.9993226389661074, + "pass@16": 0.9999261001036863, + "pass@32": 0.9999993808196772, + "pass@64": 0.999999999998295, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930061751497006, + "pass@2": 0.9955111921825641, + "pass@4": 0.9965653779617369, + "pass@8": 0.9969513016358855, + "pass@16": 0.99700529626788, + "pass@32": 0.9970059879973182, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9746799698795181, + "pass@2": 0.9820036790864243, + "pass@4": 0.9869027902950386, + "pass@8": 0.9914568557373679, + "pass@16": 0.9954504095570273, + "pass@32": 0.9981957341656477, + "pass@64": 0.9996185244010403, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9447247257053292, + "pass@2": 0.9670325728778417, + "pass@4": 0.982604836425227, + "pass@8": 0.9920601410768102, + "pass@16": 0.9971194098066611, + "pass@32": 0.999438072623504, + "pass@64": 0.9999856781124098, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9464285714285714, + "pass@2": 0.9673766652330223, + "pass@4": 0.982342608697022, + "pass@8": 0.9916849785119608, + "pass@16": 0.9966756560879082, + "pass@32": 0.9990953691874354, + "pass@64": 0.9999426917920984, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9462262426900585, + "pass@2": 0.9666012657134964, + "pass@4": 0.9802627821193401, + "pass@8": 0.989257075759594, + "pass@16": 0.9947739879169092, + "pass@32": 0.9978547681947219, + "pass@64": 0.9992660885074793, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9564393939393939, + "pass@2": 0.9771869780481988, + "pass@4": 0.9897114525457041, + "pass@8": 0.9962214241568061, + "pass@16": 0.9981651536380237, + "pass@32": 0.9990561561044676, + "pass@64": 0.9998195180147036, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8993723290598291, + "pass@2": 0.9290227443524688, + "pass@4": 0.9515008631399704, + "pass@8": 0.9674285384849971, + "pass@16": 0.9772017098299314, + "pass@32": 0.9839347977212497, + "pass@64": 0.9903181373580099, + "pass@128": 0.9971509971509972 + }, + "17": { + "pass@1": 0.8847777562111802, + "pass@2": 0.9210164541252994, + "pass@4": 0.9453830207365381, + "pass@8": 0.9613599524029054, + "pass@16": 0.9711743000764183, + "pass@32": 0.9769603297072578, + "pass@64": 0.9796747066669252, + "pass@128": 0.9813664596273292 + }, + "18": { + "pass@1": 0.8802619485294118, + "pass@2": 0.9259028340666982, + "pass@4": 0.9532336597631177, + "pass@8": 0.9707902431940827, + "pass@16": 0.9811945823460004, + "pass@32": 0.9869220810987678, + "pass@64": 0.9895681723618133, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7547665550595238, + "pass@2": 0.8166603686257972, + "pass@4": 0.8588360034794753, + "pass@8": 0.8899064276709484, + "pass@16": 0.9129267281645536, + "pass@32": 0.9315558807032726, + "pass@64": 0.9471272503883296, + "pass@128": 0.9613095238095238 + }, + "20": { + "pass@1": 0.718562874251497, + "pass@2": 0.7972754048988645, + "pass@4": 0.8476215479614443, + "pass@8": 0.8824998849887433, + "pass@16": 0.9078157775076584, + "pass@32": 0.9261277125040664, + "pass@64": 0.9398950398697754, + "pass@128": 0.9491017964071856 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 744687, + "answer_accuracy": 0.9295202408531714, + "avg_response_len": 313.48168637162485, + "resp_tokens_sum": 251146480, + "pass_at_k": { + "pass@1": 0.9295202408531714, + "pass@2": 0.9510633821155672, + "pass@4": 0.96557593465859, + "pass@8": 0.9756436851523189, + "pass@16": 0.9824857474519137, + "pass@32": 0.9873252190122862, + "pass@64": 0.991204337729149, + "pass@128": 0.9944080524045374 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9482742537313433, + "pass@2": 0.9622781760488895, + "pass@4": 0.9745147809881973, + "pass@8": 0.984917465243863, + "pass@16": 0.9919555851114118, + "pass@32": 0.9955439709907747, + "pass@64": 0.9968257197625695, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99995444606414, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 0.9999996402587834, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9902803308823529, + "pass@2": 0.9940535983094023, + "pass@4": 0.9970285755640838, + "pass@8": 0.9991619038527141, + "pass@16": 0.9999260073209658, + "pass@32": 0.9999996124045837, + "pass@64": 0.9999999999998251, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992852917981072, + "pass@2": 0.9998924929829356, + "pass@4": 0.9999968578454508, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9962668117088608, + "pass@2": 0.9977963345958338, + "pass@4": 0.9986328544374992, + "pass@8": 0.999429871140494, + "pass@16": 0.9999100287072848, + "pass@32": 0.9999986312524417, + "pass@64": 0.9999999999868562, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931024774774775, + "pass@2": 0.9961956296848028, + "pass@4": 0.9978747648098041, + "pass@8": 0.9992037248512674, + "pass@16": 0.9998744597144952, + "pass@32": 0.9999973189913167, + "pass@64": 0.9999999999193833, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9937198553054662, + "pass@2": 0.9955965801200093, + "pass@4": 0.9970144173095726, + "pass@8": 0.998524103031284, + "pass@16": 0.9995907955741878, + "pass@32": 0.9999680781115876, + "pass@64": 0.9999999611913292, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9676835317460317, + "pass@2": 0.9811679790026249, + "pass@4": 0.9908895885038183, + "pass@8": 0.9961796859892825, + "pass@16": 0.9983880215612163, + "pass@32": 0.9994515913942914, + "pass@64": 0.9999561147211625, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9483467125382263, + "pass@2": 0.9696277751691591, + "pass@4": 0.9849515017618209, + "pass@8": 0.9936798937954158, + "pass@16": 0.9973701844414204, + "pass@32": 0.9987044705498487, + "pass@64": 0.9996267656297112, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.8966531635802469, + "pass@2": 0.9322411629969862, + "pass@4": 0.9603198564299832, + "pass@8": 0.9793484492294058, + "pass@16": 0.9909017841636405, + "pass@32": 0.9974345519388546, + "pass@64": 0.9998240665268706, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9573210311572701, + "pass@2": 0.9757080334820909, + "pass@4": 0.9848944726679192, + "pass@8": 0.9903352318613627, + "pass@16": 0.993704163528141, + "pass@32": 0.9964393394993372, + "pass@64": 0.9984021166636863, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9275655864197531, + "pass@2": 0.9555214560853507, + "pass@4": 0.9713424178459172, + "pass@8": 0.9795288181559892, + "pass@16": 0.983890943047196, + "pass@32": 0.9876471077266684, + "pass@64": 0.9921078185425506, + "pass@128": 0.9969135802469136 + }, + "16": { + "pass@1": 0.8962942477876106, + "pass@2": 0.9278748722504823, + "pass@4": 0.9495447363880404, + "pass@8": 0.965958245819696, + "pass@16": 0.9772082397000456, + "pass@32": 0.9842197304599816, + "pass@64": 0.9893026678839408, + "pass@128": 0.9941002949852508 + }, + "17": { + "pass@1": 0.8444088855421686, + "pass@2": 0.8785812541504603, + "pass@4": 0.9036703933280926, + "pass@8": 0.9250707536806557, + "pass@16": 0.9421445829755859, + "pass@32": 0.9558133544887637, + "pass@64": 0.968130379274754, + "pass@128": 0.9789156626506024 + }, + "18": { + "pass@1": 0.8652721774193548, + "pass@2": 0.9117249015748031, + "pass@4": 0.9395046590950324, + "pass@8": 0.9577380278438656, + "pass@16": 0.9709860486100815, + "pass@32": 0.9810653945270422, + "pass@64": 0.9893865711937819, + "pass@128": 0.9967741935483871 + }, + "19": { + "pass@1": 0.7195524924471299, + "pass@2": 0.7916381699455244, + "pass@4": 0.8417271315405817, + "pass@8": 0.8787354158112974, + "pass@16": 0.9089822428815135, + "pass@32": 0.9344530494883864, + "pass@64": 0.9560472262738914, + "pass@128": 0.9697885196374623 + }, + "20": { + "pass@1": 0.7147147147147147, + "pass@2": 0.7995218397531393, + "pass@4": 0.854075389224995, + "pass@8": 0.8897990122214269, + "pass@16": 0.9129369733378919, + "pass@32": 0.9290925465576422, + "pass@64": 0.9439802220780451, + "pass@128": 0.960960960960961 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_id_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..75f26268a57fa11d8c69d16b6cc15eb929591017 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76683800e0ad51f1832fa22f31a9e5a8eb4ab3a876fc32be161d191acfad42f4 +size 1790186994 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_metrics.json b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b4b7205f1ce1cfc7826e1d5ba569cbf031357f --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-39008", + "id": { + "answer_accuracy": 0.9907847222222222, + "avg_response_len": 200.2925859375, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9907847222222222, + "pass@2": 0.9934238435039348, + "pass@4": 0.9950592596967054, + "pass@8": 0.9961642945843818, + "pass@16": 0.9970317432110862, + "pass@32": 0.9978001231542015, + "pass@64": 0.9984549341716281, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392421875, + "pass@2": 0.9524984005905512, + "pass@4": 0.963226276152981, + "pass@8": 0.9713286921299945, + "pass@16": 0.9777669333707508, + "pass@32": 0.9832393396563197, + "pass@64": 0.987876988783968, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99875, + "pass@2": 0.9996080216535433, + "pass@4": 0.9999247419385078, + "pass@8": 0.9999959482495361, + "pass@16": 0.9999999908354918, + "pass@32": 0.9999999999999952, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989921875, + "pass@2": 0.9990764025590552, + "pass@4": 0.9991489848143982, + "pass@8": 0.999279659004318, + "pass@16": 0.9994929298958598, + "pass@32": 0.9997689651293588, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9952421875, + "pass@2": 0.9979008366141732, + "pass@4": 0.9987994468503937, + "pass@8": 0.9990418276537036, + "pass@16": 0.9991248330446889, + "pass@32": 0.9992499999968469, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997109375, + "pass@2": 0.999976624015748, + "pass@4": 0.9999997013498313, + "pass@8": 0.9999999999693874, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99740625, + "pass@2": 0.9991331200787402, + "pass@4": 0.9997483524559431, + "pass@8": 0.9999713355352924, + "pass@16": 0.99999966709316, + "pass@32": 0.9999999999849807, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.994984375, + "pass@2": 0.9979394685039371, + "pass@4": 0.9991879809711285, + "pass@8": 0.9997312208212122, + "pass@16": 0.999952510493212, + "pass@32": 0.9999987283554818, + "pass@64": 0.9999999999331988, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992734375, + "pass@2": 0.9946817175196855, + "pass@4": 0.9954978527371577, + "pass@8": 0.9961299678959952, + "pass@16": 0.9969488241666123, + "pass@32": 0.9979440752648279, + "pass@64": 0.9987462377251222, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9392421875, + "2": 1.0, + "3": 0.99875, + "4": 0.9989921875, + "5": 0.9952421875, + "6": 0.9997109375, + "7": 0.99740625, + "8": 0.994984375, + "9": 0.992734375 + }, + "per_op_avg_response_len": { + "10": 281.4669453125, + "2": 115.5925546875, + "3": 145.275953125, + "4": 159.883703125, + "5": 173.706953125, + "6": 199.02434375, + "7": 224.1815234375, + "8": 243.8110234375, + "9": 259.6902734375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4669453125, + "2": 115.5925546875, + "3": 145.275953125, + "4": 159.883703125, + "5": 173.706953125, + "6": 199.02434375, + "7": 224.1815234375, + "8": 243.8110234375, + "9": 259.6902734375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375468, + "answer_accuracy": 0.9906598277608916, + "avg_response_len": 190.80512284701115, + "resp_tokens_sum": 72316668, + "pass_at_k": { + "pass@1": 0.9906598277608916, + "pass@2": 0.9929249202892193, + "pass@4": 0.9942894798674481, + "pass@8": 0.9953506002826571, + "pass@16": 0.9963114259877901, + "pass@32": 0.9971869416324682, + "pass@64": 0.9978994572979196, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9401963495575221, + "pass@2": 0.9506222708057512, + "pass@4": 0.9587029428290492, + "pass@8": 0.9658143010418404, + "pass@16": 0.9722315605765348, + "pass@32": 0.9776190501932904, + "pass@64": 0.9821069470948363, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998184250764526, + "pass@2": 0.9991895723470345, + "pass@4": 0.9997933970982985, + "pass@8": 0.9999877146197467, + "pass@16": 0.99999997197471, + "pass@32": 0.9999999999999853, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969482421875, + "pass@2": 0.997114911417323, + "pass@4": 0.9973405775449944, + "pass@8": 0.9977489343884937, + "pass@16": 0.9984154059245618, + "pass@32": 0.9992780160292464, + "pass@64": 0.999909940944882, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961309523809524, + "pass@2": 0.9987071928508937, + "pass@4": 0.999725366472048, + "pass@8": 0.9999813917311221, + "pass@16": 0.9999999349314345, + "pass@32": 0.9999999999998986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.9998817004845547, + "pass@4": 0.9999975042875557, + "pass@8": 0.9999999989853416, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9947916666666666, + "pass@2": 0.9978863819907128, + "pass@4": 0.9992973630700009, + "pass@8": 0.9998804787336676, + "pass@16": 0.9999966807249981, + "pass@32": 0.999999999134954, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9931117957746479, + "pass@2": 0.9946046356881446, + "pass@4": 0.9952563448935078, + "pass@8": 0.9960270148775638, + "pass@16": 0.9971824470231253, + "pass@32": 0.9985597501374238, + "pass@64": 0.9996474842017711, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386678, + "answer_accuracy": 0.9898171281127129, + "avg_response_len": 198.2944150352228, + "resp_tokens_sum": 77464903, + "pass_at_k": { + "pass@1": 0.9898171281127129, + "pass@2": 0.9926261725885187, + "pass@4": 0.9944506663587922, + "pass@8": 0.9956962915768267, + "pass@16": 0.9966490515632801, + "pass@32": 0.9975249524143441, + "pass@64": 0.9984037960861777, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9255416027607362, + "pass@2": 0.9411788681706198, + "pass@4": 0.9550019118584102, + "pass@8": 0.964868285928447, + "pass@16": 0.9719946880736567, + "pass@32": 0.9785032582118173, + "pass@64": 0.9854509352402538, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9996178567167742, + "pass@4": 0.9999766702457646, + "pass@8": 0.99999989566327, + "pass@16": 0.9999999999992779, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9975317028985508, + "pass@2": 0.9996298356727149, + "pass@4": 0.9999874317340767, + "pass@8": 0.999999982420672, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993508237822349, + "pass@2": 0.9999333726282066, + "pass@4": 0.9999991442688574, + "pass@8": 0.9999999999122846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978775289017341, + "pass@2": 0.9996525960356833, + "pass@4": 0.9999823395449846, + "pass@8": 0.99999995023541, + "pass@16": 0.9999999999998741, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9940801056338028, + "pass@2": 0.9969987246312523, + "pass@4": 0.9984681511642031, + "pass@8": 0.9993523248812373, + "pass@16": 0.9998691472743508, + "pass@32": 0.9999964186630312, + "pass@64": 0.9999999998118273, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993123128742515, + "pass@2": 0.9944116442312227, + "pass@4": 0.9948953917249864, + "pass@8": 0.9956527038685988, + "pass@16": 0.9968535622066148, + "pass@32": 0.9983693531920645, + "pass@64": 0.9996149126751299, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379238, + "answer_accuracy": 0.9918971794442585, + "avg_response_len": 211.73911951791095, + "resp_tokens_sum": 80955488, + "pass_at_k": { + "pass@1": 0.9918971794442585, + "pass@2": 0.9947334529022087, + "pass@4": 0.9964441759811826, + "pass@8": 0.9974490933813244, + "pass@16": 0.998136809902511, + "pass@32": 0.9986891260279526, + "pass@64": 0.9990578268598913, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9516091417910447, + "pass@2": 0.9654123575038193, + "pass@4": 0.9758070306510193, + "pass@8": 0.9831957697436913, + "pass@16": 0.9889855702187899, + "pass@32": 0.9935355804887811, + "pass@64": 0.9960768024790906, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999974891531417, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999086257309941, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920955882352941, + "pass@2": 0.9953993457619268, + "pass@4": 0.9967361543042413, + "pass@8": 0.9971991038919662, + "pass@16": 0.9974260398273237, + "pass@32": 0.997794117637879, + "pass@64": 0.9985294117647059, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9977636300209309, + "pass@4": 0.9992256535575459, + "pass@8": 0.9999093452430227, + "pass@16": 0.9999989464974799, + "pass@32": 0.999999999952471, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9961289414414415, + "pass@2": 0.998992102338559, + "pass@4": 0.9998528829729619, + "pass@8": 0.9999953036740797, + "pass@16": 0.9999999964703241, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9918860530546624, + "pass@2": 0.995059751373522, + "pass@4": 0.9964205448997332, + "pass@8": 0.9967600466956527, + "pass@16": 0.9967844556154133, + "pass@32": 0.996784565916054, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.87380859375, + "avg_response_len": 381.57111484375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.87380859375, + "pass@2": 0.9130243602361937, + "pass@4": 0.9378349559336265, + "pass@8": 0.954473771465677, + "pass@16": 0.9660566269187735, + "pass@32": 0.9744658616525587, + "pass@64": 0.9805980411365114, + "pass@128": 0.985 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9660859375, + "pass@2": 0.9788811515748035, + "pass@4": 0.9871767390326209, + "pass@8": 0.992585784423573, + "pass@16": 0.996207095656206, + "pass@32": 0.9984438082497634, + "pass@64": 0.9994463945531495, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.941984375, + "pass@2": 0.9640142716535431, + "pass@4": 0.9799431273903264, + "pass@8": 0.9897628547971873, + "pass@16": 0.9952973782211142, + "pass@32": 0.9982456186228615, + "pass@64": 0.9994235816742779, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.917765625, + "pass@2": 0.9470322342519685, + "pass@4": 0.9674209870641167, + "pass@8": 0.9810722717421293, + "pass@16": 0.9896790634487782, + "pass@32": 0.9940260078196764, + "pass@64": 0.9961032234504201, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.9350625, + "pass@2": 0.958563484251968, + "pass@4": 0.9741016163292094, + "pass@8": 0.9843825898641114, + "pass@16": 0.9907027954683493, + "pass@32": 0.9945314127488786, + "pass@64": 0.9966956292667858, + "pass@128": 0.998 + }, + "15": { + "pass@1": 0.9491953125, + "pass@2": 0.9719883120078749, + "pass@4": 0.9830726247656538, + "pass@8": 0.9894175091591041, + "pass@16": 0.9927580067424796, + "pass@32": 0.994399988681819, + "pass@64": 0.9954408365176881, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.8780078125, + "pass@2": 0.9127250246062999, + "pass@4": 0.9351123285526808, + "pass@8": 0.9504058271663447, + "pass@16": 0.9612878577546778, + "pass@32": 0.9697369867676892, + "pass@64": 0.9761366690097133, + "pass@128": 0.981 + }, + "17": { + "pass@1": 0.87325, + "pass@2": 0.9060204232283476, + "pass@4": 0.9257824860329958, + "pass@8": 0.940537455214353, + "pass@16": 0.9525379374572871, + "pass@32": 0.9625648410706185, + "pass@64": 0.9708393346950811, + "pass@128": 0.977 + }, + "18": { + "pass@1": 0.846953125, + "pass@2": 0.8975238681102372, + "pass@4": 0.9266614673790771, + "pass@8": 0.9459521533095834, + "pass@16": 0.959952081234358, + "pass@32": 0.9704434978893687, + "pass@64": 0.9783195372717939, + "pass@128": 0.984 + }, + "19": { + "pass@1": 0.727203125, + "pass@2": 0.8011141732283465, + "pass@4": 0.850858918916386, + "pass@8": 0.8861175894209639, + "pass@16": 0.9118022791650074, + "pass@32": 0.9309020379871955, + "pass@64": 0.9443730079238775, + "pass@128": 0.953 + }, + "20": { + "pass@1": 0.702578125, + "pass@2": 0.7923806594488186, + "pass@4": 0.8482192638732657, + "pass@8": 0.884503679559428, + "pass@16": 0.9103417740394238, + "pass@32": 0.9313644166876854, + "pass@64": 0.9492021970023096, + "pass@128": 0.963 + } + }, + "per_op_accuracy": { + "11": 0.9660859375, + "12": 0.941984375, + "13": 0.917765625, + "14": 0.9350625, + "15": 0.9491953125, + "16": 0.8780078125, + "17": 0.87325, + "18": 0.846953125, + "19": 0.727203125, + "20": 0.702578125 + }, + "per_op_avg_response_len": { + "11": 300.84703125, + "12": 313.212640625, + "13": 334.147953125, + "14": 361.053375, + "15": 381.114171875, + "16": 391.4247421875, + "17": 412.8846875, + "18": 430.6323671875, + "19": 434.444859375, + "20": 455.9493203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 300.84703125, + "12": 313.212640625, + "13": 334.147953125, + "14": 361.053375, + "15": 381.114171875, + "16": 391.4247421875, + "17": 412.8846875, + "18": 430.6323671875, + "19": 434.444859375, + "20": 455.9493203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 374986, + "answer_accuracy": 0.8706027117384844, + "avg_response_len": 363.53099693536404, + "resp_tokens_sum": 156580071, + "pass_at_k": { + "pass@1": 0.8706027117384844, + "pass@2": 0.9096660343859367, + "pass@4": 0.9341743568829554, + "pass@8": 0.9504782920011838, + "pass@16": 0.9621234444836287, + "pass@32": 0.9713529360839749, + "pass@64": 0.9785262105330861, + "pass@128": 0.9830609212481426 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.961003895184136, + "pass@2": 0.9758757193682941, + "pass@4": 0.9859773047136813, + "pass@8": 0.9925369319619592, + "pass@16": 0.996719964796309, + "pass@32": 0.9992331687805646, + "pass@64": 0.9999750061727128, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9469014830508474, + "pass@2": 0.9680802777036346, + "pass@4": 0.9823547948985191, + "pass@8": 0.9907254129910008, + "pass@16": 0.9957055694665387, + "pass@32": 0.998830769032777, + "pass@64": 0.9999113679009122, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9084541536050157, + "pass@2": 0.9421539845234863, + "pass@4": 0.9648193210762447, + "pass@8": 0.9782534132080241, + "pass@16": 0.9858754146175864, + "pass@32": 0.9896348514061588, + "pass@64": 0.9917705113956121, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.911117601246106, + "pass@2": 0.9386605587852919, + "pass@4": 0.9593386028265156, + "pass@8": 0.973756274172, + "pass@16": 0.9835840404357657, + "pass@32": 0.9901129291802138, + "pass@64": 0.9944569067911632, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9607794436416185, + "pass@2": 0.9809912782759094, + "pass@4": 0.9878286179726087, + "pass@8": 0.9906328734957064, + "pass@16": 0.9918557494374052, + "pass@32": 0.992594484098195, + "pass@64": 0.9935027989512581, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8590725806451613, + "pass@2": 0.8959617887985772, + "pass@4": 0.92299481324915, + "pass@8": 0.9429119820308187, + "pass@16": 0.9569669200321872, + "pass@32": 0.9684234527512353, + "pass@64": 0.9777462994805372, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8935603323699421, + "pass@2": 0.9237261617586823, + "pass@4": 0.9405292829183923, + "pass@8": 0.9529428599398514, + "pass@16": 0.963253636094547, + "pass@32": 0.9715690776732758, + "pass@64": 0.9776152235402346, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8213169642857143, + "pass@2": 0.8683225534308209, + "pass@4": 0.8961326584176972, + "pass@8": 0.9165233758627166, + "pass@16": 0.9336095470655086, + "pass@32": 0.9487447437891168, + "pass@64": 0.9618053813005009, + "pass@128": 0.9714285714285714 + }, + "19": { + "pass@1": 0.7183511636636637, + "pass@2": 0.7900654098980873, + "pass@4": 0.8403002755286215, + "pass@8": 0.8751205968307463, + "pass@16": 0.8989661660960092, + "pass@32": 0.917543447331758, + "pass@64": 0.932127476264556, + "pass@128": 0.93993993993994 + }, + "20": { + "pass@1": 0.7155827702702703, + "pass@2": 0.8054037058712256, + "pass@4": 0.8563406066358814, + "pass@8": 0.8878686428984232, + "pass@16": 0.9121683780580442, + "pass@32": 0.9349939045842077, + "pass@64": 0.9550958412627336, + "pass@128": 0.9669669669669669 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 379446, + "answer_accuracy": 0.8814813782337199, + "avg_response_len": 377.0042419342849, + "resp_tokens_sum": 162286754, + "pass_at_k": { + "pass@1": 0.8814813782337199, + "pass@2": 0.9186794370652425, + "pass@4": 0.9426408757335323, + "pass@8": 0.9590559032865142, + "pass@16": 0.9703121290667686, + "pass@32": 0.9781364887007047, + "pass@64": 0.9837972578745944, + "pass@128": 0.9887005649717514 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9734563253012049, + "pass@2": 0.9823694383834549, + "pass@4": 0.987195932210281, + "pass@8": 0.9907554712427886, + "pass@16": 0.9940552664938294, + "pass@32": 0.9966879425459982, + "pass@64": 0.998400809093451, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9348795062695925, + "pass@2": 0.9590008145533532, + "pass@4": 0.976941083579286, + "pass@8": 0.9878423646843028, + "pass@16": 0.9936105268573175, + "pass@32": 0.9967195694500087, + "pass@64": 0.9983826288179706, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9399728641456583, + "pass@2": 0.961666556386334, + "pass@4": 0.9770743410224988, + "pass@8": 0.9880556844662887, + "pass@16": 0.9949278244154547, + "pass@32": 0.9980192365368127, + "pass@64": 0.999299221759988, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9399442616959064, + "pass@2": 0.9629545690012435, + "pass@4": 0.979254098720116, + "pass@8": 0.9904509185434833, + "pass@16": 0.9963750780080175, + "pass@32": 0.9990808958940353, + "pass@64": 0.9999511240349822, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9578361742424243, + "pass@2": 0.9781145609639704, + "pass@4": 0.9891054819852062, + "pass@8": 0.9964601309728172, + "pass@16": 0.9994909800349003, + "pass@32": 0.9999845859908743, + "pass@64": 0.9999999969188734, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8879763176638177, + "pass@2": 0.918719294703547, + "pass@4": 0.9389211383299313, + "pass@8": 0.9522550224776325, + "pass@16": 0.9622880180618023, + "pass@32": 0.9711609398649986, + "pass@64": 0.9784153797441794, + "pass@128": 0.9857549857549858 + }, + "17": { + "pass@1": 0.8806531444099379, + "pass@2": 0.9134255636523697, + "pass@4": 0.9319340411447012, + "pass@8": 0.9455538506849092, + "pass@16": 0.9575326946061652, + "pass@32": 0.9686242097805646, + "pass@64": 0.9789813555707505, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8600183823529411, + "pass@2": 0.9148224004168601, + "pass@4": 0.946246044795871, + "pass@8": 0.965380451817509, + "pass@16": 0.9765385504168345, + "pass@32": 0.9824988391462636, + "pass@64": 0.9867292851981442, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7423735119047619, + "pass@2": 0.8135465000468688, + "pass@4": 0.8603529316089945, + "pass@8": 0.8940199961295004, + "pass@16": 0.9191097672675497, + "pass@32": 0.9370573825963017, + "pass@64": 0.9482075383975361, + "pass@128": 0.9553571428571429 + }, + "20": { + "pass@1": 0.6969966317365269, + "pass@2": 0.7815660509689274, + "pass@4": 0.8387318512581132, + "pass@8": 0.8791036231156621, + "pass@16": 0.908555207050984, + "pass@32": 0.9310486628748871, + "pass@64": 0.9493486850175739, + "pass@128": 0.9670658682634731 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 364043, + "answer_accuracy": 0.8692194185513448, + "avg_response_len": 404.8178722875917, + "resp_tokens_sum": 169544202, + "pass_at_k": { + "pass@1": 0.8692194185513448, + "pass@2": 0.9106657853921639, + "pass@4": 0.9366600193561564, + "pass@8": 0.953873245513524, + "pass@16": 0.9657277470809144, + "pass@32": 0.9738945522929429, + "pass@64": 0.9794405668364868, + "pass@128": 0.9831907090464548 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9640128968253968, + "pass@2": 0.9785726002999626, + "pass@4": 0.9885006380154859, + "pass@8": 0.9945696221853831, + "pass@16": 0.9979003162132001, + "pass@32": 0.9994098499839769, + "pass@64": 0.9999560277941457, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9435923165137615, + "pass@2": 0.9645033440487374, + "pass@4": 0.9802609306864158, + "pass@8": 0.9905943249666102, + "pass@16": 0.99650106594029, + "pass@32": 0.9991008676779993, + "pass@64": 0.9999110056404356, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9024643132716049, + "pass@2": 0.9357103474044912, + "pass@4": 0.9593459379498855, + "pass@8": 0.9761529431305727, + "pass@16": 0.9876406570044788, + "pass@32": 0.993949453016883, + "pass@64": 0.9968475560089626, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9529163575667656, + "pass@2": 0.9730652293277879, + "pass@4": 0.9829347865270549, + "pass@8": 0.9883460288220425, + "pass@16": 0.9917271270318293, + "pass@32": 0.9941231337871526, + "pass@64": 0.9955242663704985, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9280237268518519, + "pass@2": 0.9561343352046277, + "pass@4": 0.9718491169506588, + "pass@8": 0.9809465793474069, + "pass@16": 0.9868638704494458, + "pass@32": 0.9906400734779464, + "pass@64": 0.9928668799300133, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8850018436578171, + "pass@2": 0.9218477806424639, + "pass@4": 0.9422496073499659, + "pass@8": 0.9553439523514501, + "pass@16": 0.9642035935250982, + "pass@32": 0.9694637950507127, + "pass@64": 0.9723053624794668, + "pass@128": 0.9734513274336283 + }, + "17": { + "pass@1": 0.8449030496987951, + "pass@2": 0.8803859631439148, + "pass@4": 0.9044475689597533, + "pass@8": 0.9227436317910335, + "pass@16": 0.9365260593535809, + "pass@32": 0.9473040501395271, + "pass@64": 0.9558809363143925, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.8615675403225806, + "pass@2": 0.9115205105410208, + "pass@4": 0.9396496183944743, + "pass@8": 0.9578697359344478, + "pass@16": 0.971502040708731, + "pass@32": 0.9817201040433834, + "pass@64": 0.9877409575782228, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7207090256797583, + "pass@2": 0.7996095701405903, + "pass@4": 0.8518439339719992, + "pass@8": 0.8891592506852346, + "pass@16": 0.9172980787103946, + "pass@32": 0.938093019556988, + "pass@64": 0.952800076816336, + "pass@128": 0.9637462235649547 + }, + "20": { + "pass@1": 0.6951717342342343, + "pass@2": 0.7902046978080438, + "pass@4": 0.8496138244543745, + "pass@8": 0.8865549890499133, + "pass@16": 0.9103071020752801, + "pass@32": 0.928051630813609, + "pass@64": 0.9431616248226694, + "pass@128": 0.954954954954955 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.9292183388157895, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9292183388157895, + "pass@2": 0.9511083259946042, + "pass@4": 0.9649412050845557, + "pass@8": 0.9742219139955987, + "pass@16": 0.9807290504256501, + "pass@32": 0.9855189328901838, + "pass@64": 0.989056569416302, + "pass@128": 0.9916315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392421875, + "pass@2": 0.9524984005905512, + "pass@4": 0.963226276152981, + "pass@8": 0.9713286921299945, + "pass@16": 0.9777669333707508, + "pass@32": 0.9832393396563197, + "pass@64": 0.987876988783968, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99875, + "pass@2": 0.9996080216535433, + "pass@4": 0.9999247419385078, + "pass@8": 0.9999959482495361, + "pass@16": 0.9999999908354918, + "pass@32": 0.9999999999999952, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989921875, + "pass@2": 0.9990764025590552, + "pass@4": 0.9991489848143982, + "pass@8": 0.999279659004318, + "pass@16": 0.9994929298958598, + "pass@32": 0.9997689651293588, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9952421875, + "pass@2": 0.9979008366141732, + "pass@4": 0.9987994468503937, + "pass@8": 0.9990418276537036, + "pass@16": 0.9991248330446889, + "pass@32": 0.9992499999968469, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997109375, + "pass@2": 0.999976624015748, + "pass@4": 0.9999997013498313, + "pass@8": 0.9999999999693874, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99740625, + "pass@2": 0.9991331200787402, + "pass@4": 0.9997483524559431, + "pass@8": 0.9999713355352924, + "pass@16": 0.99999966709316, + "pass@32": 0.9999999999849807, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.994984375, + "pass@2": 0.9979394685039371, + "pass@4": 0.9991879809711285, + "pass@8": 0.9997312208212122, + "pass@16": 0.999952510493212, + "pass@32": 0.9999987283554818, + "pass@64": 0.9999999999331988, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992734375, + "pass@2": 0.9946817175196855, + "pass@4": 0.9954978527371577, + "pass@8": 0.9961299678959952, + "pass@16": 0.9969488241666123, + "pass@32": 0.9979440752648279, + "pass@64": 0.9987462377251222, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9660859375, + "pass@2": 0.9788811515748035, + "pass@4": 0.9871767390326209, + "pass@8": 0.992585784423573, + "pass@16": 0.996207095656206, + "pass@32": 0.9984438082497634, + "pass@64": 0.9994463945531495, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.941984375, + "pass@2": 0.9640142716535431, + "pass@4": 0.9799431273903264, + "pass@8": 0.9897628547971873, + "pass@16": 0.9952973782211142, + "pass@32": 0.9982456186228615, + "pass@64": 0.9994235816742779, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.917765625, + "pass@2": 0.9470322342519685, + "pass@4": 0.9674209870641167, + "pass@8": 0.9810722717421293, + "pass@16": 0.9896790634487782, + "pass@32": 0.9940260078196764, + "pass@64": 0.9961032234504201, + "pass@128": 0.998 + }, + "14": { + "pass@1": 0.9350625, + "pass@2": 0.958563484251968, + "pass@4": 0.9741016163292094, + "pass@8": 0.9843825898641114, + "pass@16": 0.9907027954683493, + "pass@32": 0.9945314127488786, + "pass@64": 0.9966956292667858, + "pass@128": 0.998 + }, + "15": { + "pass@1": 0.9491953125, + "pass@2": 0.9719883120078749, + "pass@4": 0.9830726247656538, + "pass@8": 0.9894175091591041, + "pass@16": 0.9927580067424796, + "pass@32": 0.994399988681819, + "pass@64": 0.9954408365176881, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.8780078125, + "pass@2": 0.9127250246062999, + "pass@4": 0.9351123285526808, + "pass@8": 0.9504058271663447, + "pass@16": 0.9612878577546778, + "pass@32": 0.9697369867676892, + "pass@64": 0.9761366690097133, + "pass@128": 0.981 + }, + "17": { + "pass@1": 0.87325, + "pass@2": 0.9060204232283476, + "pass@4": 0.9257824860329958, + "pass@8": 0.940537455214353, + "pass@16": 0.9525379374572871, + "pass@32": 0.9625648410706185, + "pass@64": 0.9708393346950811, + "pass@128": 0.977 + }, + "18": { + "pass@1": 0.846953125, + "pass@2": 0.8975238681102372, + "pass@4": 0.9266614673790771, + "pass@8": 0.9459521533095834, + "pass@16": 0.959952081234358, + "pass@32": 0.9704434978893687, + "pass@64": 0.9783195372717939, + "pass@128": 0.984 + }, + "19": { + "pass@1": 0.727203125, + "pass@2": 0.8011141732283465, + "pass@4": 0.850858918916386, + "pass@8": 0.8861175894209639, + "pass@16": 0.9118022791650074, + "pass@32": 0.9309020379871955, + "pass@64": 0.9443730079238775, + "pass@128": 0.953 + }, + "20": { + "pass@1": 0.702578125, + "pass@2": 0.7923806594488186, + "pass@4": 0.8482192638732657, + "pass@8": 0.884503679559428, + "pass@16": 0.9103417740394238, + "pass@32": 0.9313644166876854, + "pass@64": 0.9492021970023096, + "pass@128": 0.963 + } + }, + "per_op_accuracy": { + "10": 0.9392421875, + "2": 1.0, + "3": 0.99875, + "4": 0.9989921875, + "5": 0.9952421875, + "6": 0.9997109375, + "7": 0.99740625, + "8": 0.994984375, + "9": 0.992734375, + "11": 0.9660859375, + "12": 0.941984375, + "13": 0.917765625, + "14": 0.9350625, + "15": 0.9491953125, + "16": 0.8780078125, + "17": 0.87325, + "18": 0.846953125, + "19": 0.727203125, + "20": 0.702578125 + }, + "per_op_avg_response_len": { + "10": 281.4669453125, + "2": 115.5925546875, + "3": 145.275953125, + "4": 159.883703125, + "5": 173.706953125, + "6": 199.02434375, + "7": 224.1815234375, + "8": 243.8110234375, + "9": 259.6902734375, + "11": 300.84703125, + "12": 313.212640625, + "13": 334.147953125, + "14": 361.053375, + "15": 381.114171875, + "16": 391.4247421875, + "17": 412.8846875, + "18": 430.6323671875, + "19": 434.444859375, + "20": 455.9493203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4669453125, + "2": 115.5925546875, + "3": 145.275953125, + "4": 159.883703125, + "5": 173.706953125, + "6": 199.02434375, + "7": 224.1815234375, + "8": 243.8110234375, + "9": 259.6902734375, + "11": 300.84703125, + "12": 313.212640625, + "13": 334.147953125, + "14": 361.053375, + "15": 381.114171875, + "16": 391.4247421875, + "17": 412.8846875, + "18": 430.6323671875, + "19": 434.444859375, + "20": 455.9493203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 750454, + "answer_accuracy": 0.9267976406892191, + "avg_response_len": 282.68349248142584, + "resp_tokens_sum": 228896739, + "pass_at_k": { + "pass@1": 0.9267976406892191, + "pass@2": 0.9486368787045444, + "pass@4": 0.9623123396773068, + "pass@8": 0.9714815965888218, + "pass@16": 0.9781257545111052, + "pass@32": 0.9834450148745346, + "pass@64": 0.9875942130102727, + "pass@128": 0.9903572557698388 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9401963495575221, + "pass@2": 0.9506222708057512, + "pass@4": 0.9587029428290492, + "pass@8": 0.9658143010418404, + "pass@16": 0.9722315605765348, + "pass@32": 0.9776190501932904, + "pass@64": 0.9821069470948363, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998184250764526, + "pass@2": 0.9991895723470345, + "pass@4": 0.9997933970982985, + "pass@8": 0.9999877146197467, + "pass@16": 0.99999997197471, + "pass@32": 0.9999999999999853, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969482421875, + "pass@2": 0.997114911417323, + "pass@4": 0.9973405775449944, + "pass@8": 0.9977489343884937, + "pass@16": 0.9984154059245618, + "pass@32": 0.9992780160292464, + "pass@64": 0.999909940944882, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961309523809524, + "pass@2": 0.9987071928508937, + "pass@4": 0.999725366472048, + "pass@8": 0.9999813917311221, + "pass@16": 0.9999999349314345, + "pass@32": 0.9999999999998986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.9998817004845547, + "pass@4": 0.9999975042875557, + "pass@8": 0.9999999989853416, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9947916666666666, + "pass@2": 0.9978863819907128, + "pass@4": 0.9992973630700009, + "pass@8": 0.9998804787336676, + "pass@16": 0.9999966807249981, + "pass@32": 0.999999999134954, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9931117957746479, + "pass@2": 0.9946046356881446, + "pass@4": 0.9952563448935078, + "pass@8": 0.9960270148775638, + "pass@16": 0.9971824470231253, + "pass@32": 0.9985597501374238, + "pass@64": 0.9996474842017711, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.961003895184136, + "pass@2": 0.9758757193682941, + "pass@4": 0.9859773047136813, + "pass@8": 0.9925369319619592, + "pass@16": 0.996719964796309, + "pass@32": 0.9992331687805646, + "pass@64": 0.9999750061727128, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9469014830508474, + "pass@2": 0.9680802777036346, + "pass@4": 0.9823547948985191, + "pass@8": 0.9907254129910008, + "pass@16": 0.9957055694665387, + "pass@32": 0.998830769032777, + "pass@64": 0.9999113679009122, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9084541536050157, + "pass@2": 0.9421539845234863, + "pass@4": 0.9648193210762447, + "pass@8": 0.9782534132080241, + "pass@16": 0.9858754146175864, + "pass@32": 0.9896348514061588, + "pass@64": 0.9917705113956121, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.911117601246106, + "pass@2": 0.9386605587852919, + "pass@4": 0.9593386028265156, + "pass@8": 0.973756274172, + "pass@16": 0.9835840404357657, + "pass@32": 0.9901129291802138, + "pass@64": 0.9944569067911632, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9607794436416185, + "pass@2": 0.9809912782759094, + "pass@4": 0.9878286179726087, + "pass@8": 0.9906328734957064, + "pass@16": 0.9918557494374052, + "pass@32": 0.992594484098195, + "pass@64": 0.9935027989512581, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8590725806451613, + "pass@2": 0.8959617887985772, + "pass@4": 0.92299481324915, + "pass@8": 0.9429119820308187, + "pass@16": 0.9569669200321872, + "pass@32": 0.9684234527512353, + "pass@64": 0.9777462994805372, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8935603323699421, + "pass@2": 0.9237261617586823, + "pass@4": 0.9405292829183923, + "pass@8": 0.9529428599398514, + "pass@16": 0.963253636094547, + "pass@32": 0.9715690776732758, + "pass@64": 0.9776152235402346, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8213169642857143, + "pass@2": 0.8683225534308209, + "pass@4": 0.8961326584176972, + "pass@8": 0.9165233758627166, + "pass@16": 0.9336095470655086, + "pass@32": 0.9487447437891168, + "pass@64": 0.9618053813005009, + "pass@128": 0.9714285714285714 + }, + "19": { + "pass@1": 0.7183511636636637, + "pass@2": 0.7900654098980873, + "pass@4": 0.8403002755286215, + "pass@8": 0.8751205968307463, + "pass@16": 0.8989661660960092, + "pass@32": 0.917543447331758, + "pass@64": 0.932127476264556, + "pass@128": 0.93993993993994 + }, + "20": { + "pass@1": 0.7155827702702703, + "pass@2": 0.8054037058712256, + "pass@4": 0.8563406066358814, + "pass@8": 0.8878686428984232, + "pass@16": 0.9121683780580442, + "pass@32": 0.9349939045842077, + "pass@64": 0.9550958412627336, + "pass@128": 0.9669669669669669 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 766124, + "answer_accuracy": 0.9330231878409977, + "avg_response_len": 291.9812658320343, + "resp_tokens_sum": 239751657, + "pass_at_k": { + "pass@1": 0.9330231878409977, + "pass@2": 0.953860331346917, + "pass@4": 0.9672898984908648, + "pass@8": 0.9764879321348401, + "pass@16": 0.9828421816715034, + "pass@32": 0.9873607429881609, + "pass@64": 0.9907464635833655, + "pass@128": 0.9936087295401403 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9255416027607362, + "pass@2": 0.9411788681706198, + "pass@4": 0.9550019118584102, + "pass@8": 0.964868285928447, + "pass@16": 0.9719946880736567, + "pass@32": 0.9785032582118173, + "pass@64": 0.9854509352402538, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9996178567167742, + "pass@4": 0.9999766702457646, + "pass@8": 0.99999989566327, + "pass@16": 0.9999999999992779, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9975317028985508, + "pass@2": 0.9996298356727149, + "pass@4": 0.9999874317340767, + "pass@8": 0.999999982420672, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993508237822349, + "pass@2": 0.9999333726282066, + "pass@4": 0.9999991442688574, + "pass@8": 0.9999999999122846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978775289017341, + "pass@2": 0.9996525960356833, + "pass@4": 0.9999823395449846, + "pass@8": 0.99999995023541, + "pass@16": 0.9999999999998741, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9940801056338028, + "pass@2": 0.9969987246312523, + "pass@4": 0.9984681511642031, + "pass@8": 0.9993523248812373, + "pass@16": 0.9998691472743508, + "pass@32": 0.9999964186630312, + "pass@64": 0.9999999998118273, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993123128742515, + "pass@2": 0.9944116442312227, + "pass@4": 0.9948953917249864, + "pass@8": 0.9956527038685988, + "pass@16": 0.9968535622066148, + "pass@32": 0.9983693531920645, + "pass@64": 0.9996149126751299, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9734563253012049, + "pass@2": 0.9823694383834549, + "pass@4": 0.987195932210281, + "pass@8": 0.9907554712427886, + "pass@16": 0.9940552664938294, + "pass@32": 0.9966879425459982, + "pass@64": 0.998400809093451, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9348795062695925, + "pass@2": 0.9590008145533532, + "pass@4": 0.976941083579286, + "pass@8": 0.9878423646843028, + "pass@16": 0.9936105268573175, + "pass@32": 0.9967195694500087, + "pass@64": 0.9983826288179706, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9399728641456583, + "pass@2": 0.961666556386334, + "pass@4": 0.9770743410224988, + "pass@8": 0.9880556844662887, + "pass@16": 0.9949278244154547, + "pass@32": 0.9980192365368127, + "pass@64": 0.999299221759988, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9399442616959064, + "pass@2": 0.9629545690012435, + "pass@4": 0.979254098720116, + "pass@8": 0.9904509185434833, + "pass@16": 0.9963750780080175, + "pass@32": 0.9990808958940353, + "pass@64": 0.9999511240349822, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9578361742424243, + "pass@2": 0.9781145609639704, + "pass@4": 0.9891054819852062, + "pass@8": 0.9964601309728172, + "pass@16": 0.9994909800349003, + "pass@32": 0.9999845859908743, + "pass@64": 0.9999999969188734, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8879763176638177, + "pass@2": 0.918719294703547, + "pass@4": 0.9389211383299313, + "pass@8": 0.9522550224776325, + "pass@16": 0.9622880180618023, + "pass@32": 0.9711609398649986, + "pass@64": 0.9784153797441794, + "pass@128": 0.9857549857549858 + }, + "17": { + "pass@1": 0.8806531444099379, + "pass@2": 0.9134255636523697, + "pass@4": 0.9319340411447012, + "pass@8": 0.9455538506849092, + "pass@16": 0.9575326946061652, + "pass@32": 0.9686242097805646, + "pass@64": 0.9789813555707505, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8600183823529411, + "pass@2": 0.9148224004168601, + "pass@4": 0.946246044795871, + "pass@8": 0.965380451817509, + "pass@16": 0.9765385504168345, + "pass@32": 0.9824988391462636, + "pass@64": 0.9867292851981442, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7423735119047619, + "pass@2": 0.8135465000468688, + "pass@4": 0.8603529316089945, + "pass@8": 0.8940199961295004, + "pass@16": 0.9191097672675497, + "pass@32": 0.9370573825963017, + "pass@64": 0.9482075383975361, + "pass@128": 0.9553571428571429 + }, + "20": { + "pass@1": 0.6969966317365269, + "pass@2": 0.7815660509689274, + "pass@4": 0.8387318512581132, + "pass@8": 0.8791036231156621, + "pass@16": 0.908555207050984, + "pass@32": 0.9310486628748871, + "pass@64": 0.9493486850175739, + "pass@128": 0.9670658682634731 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 743281, + "answer_accuracy": 0.9277652680140598, + "avg_response_len": 312.67436141955585, + "resp_tokens_sum": 250499690, + "pass_at_k": { + "pass@1": 0.9277652680140598, + "pass@2": 0.9507856324687619, + "pass@4": 0.9651909789086295, + "pass@8": 0.9746690687410497, + "pass@16": 0.9811944143836966, + "pass@32": 0.9857273357641817, + "pass@64": 0.9888025664674049, + "pass@128": 0.990893113915961 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9516091417910447, + "pass@2": 0.9654123575038193, + "pass@4": 0.9758070306510193, + "pass@8": 0.9831957697436913, + "pass@16": 0.9889855702187899, + "pass@32": 0.9935355804887811, + "pass@64": 0.9960768024790906, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999974891531417, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999086257309941, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920955882352941, + "pass@2": 0.9953993457619268, + "pass@4": 0.9967361543042413, + "pass@8": 0.9971991038919662, + "pass@16": 0.9974260398273237, + "pass@32": 0.997794117637879, + "pass@64": 0.9985294117647059, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9977636300209309, + "pass@4": 0.9992256535575459, + "pass@8": 0.9999093452430227, + "pass@16": 0.9999989464974799, + "pass@32": 0.999999999952471, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9961289414414415, + "pass@2": 0.998992102338559, + "pass@4": 0.9998528829729619, + "pass@8": 0.9999953036740797, + "pass@16": 0.9999999964703241, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9918860530546624, + "pass@2": 0.995059751373522, + "pass@4": 0.9964205448997332, + "pass@8": 0.9967600466956527, + "pass@16": 0.9967844556154133, + "pass@32": 0.996784565916054, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9640128968253968, + "pass@2": 0.9785726002999626, + "pass@4": 0.9885006380154859, + "pass@8": 0.9945696221853831, + "pass@16": 0.9979003162132001, + "pass@32": 0.9994098499839769, + "pass@64": 0.9999560277941457, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9435923165137615, + "pass@2": 0.9645033440487374, + "pass@4": 0.9802609306864158, + "pass@8": 0.9905943249666102, + "pass@16": 0.99650106594029, + "pass@32": 0.9991008676779993, + "pass@64": 0.9999110056404356, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9024643132716049, + "pass@2": 0.9357103474044912, + "pass@4": 0.9593459379498855, + "pass@8": 0.9761529431305727, + "pass@16": 0.9876406570044788, + "pass@32": 0.993949453016883, + "pass@64": 0.9968475560089626, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9529163575667656, + "pass@2": 0.9730652293277879, + "pass@4": 0.9829347865270549, + "pass@8": 0.9883460288220425, + "pass@16": 0.9917271270318293, + "pass@32": 0.9941231337871526, + "pass@64": 0.9955242663704985, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9280237268518519, + "pass@2": 0.9561343352046277, + "pass@4": 0.9718491169506588, + "pass@8": 0.9809465793474069, + "pass@16": 0.9868638704494458, + "pass@32": 0.9906400734779464, + "pass@64": 0.9928668799300133, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8850018436578171, + "pass@2": 0.9218477806424639, + "pass@4": 0.9422496073499659, + "pass@8": 0.9553439523514501, + "pass@16": 0.9642035935250982, + "pass@32": 0.9694637950507127, + "pass@64": 0.9723053624794668, + "pass@128": 0.9734513274336283 + }, + "17": { + "pass@1": 0.8449030496987951, + "pass@2": 0.8803859631439148, + "pass@4": 0.9044475689597533, + "pass@8": 0.9227436317910335, + "pass@16": 0.9365260593535809, + "pass@32": 0.9473040501395271, + "pass@64": 0.9558809363143925, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.8615675403225806, + "pass@2": 0.9115205105410208, + "pass@4": 0.9396496183944743, + "pass@8": 0.9578697359344478, + "pass@16": 0.971502040708731, + "pass@32": 0.9817201040433834, + "pass@64": 0.9877409575782228, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7207090256797583, + "pass@2": 0.7996095701405903, + "pass@4": 0.8518439339719992, + "pass@8": 0.8891592506852346, + "pass@16": 0.9172980787103946, + "pass@32": 0.938093019556988, + "pass@64": 0.952800076816336, + "pass@128": 0.9637462235649547 + }, + "20": { + "pass@1": 0.6951717342342343, + "pass@2": 0.7902046978080438, + "pass@4": 0.8496138244543745, + "pass@8": 0.8865549890499133, + "pass@16": 0.9103071020752801, + "pass@32": 0.928051630813609, + "pass@64": 0.9431616248226694, + "pass@128": 0.954954954954955 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_ood_generations.jsonl b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e0e01d3ca192219e4d097f64747d2397bec20f76 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfec2d2aa7ed06dcdd33792505f25efe9dca26babb3eaef92b12f892ff1d3a1d +size 2722293280 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/summary.csv b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..4b4fc5719bac3a29ba5dbb7ed946d6f08996e8b3 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-39008,0.9907847222222222,0.87380859375,0.9292183388157895,0.0,0.0,0.0,200.2925859375,381.57111484375,1152000,1280000,2432000 diff --git a/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/summary.json b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e82ed507d0803406f561985d0dfba3741499b9e6 --- /dev/null +++ b/SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-39008", + "id_acc": 0.9907847222222222, + "ood_acc": 0.87380859375, + "total_acc": 0.9292183388157895, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.2925859375, + "ood_avg_resp_len": 381.57111484375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 750454, + "answer_accuracy": 0.9267976406892191, + "avg_response_len": 282.68349248142584, + "resp_tokens_sum": 228896739, + "pass_at_k": { + "pass@1": 0.9267976406892191, + "pass@2": 0.9486368787045444, + "pass@4": 0.9623123396773068, + "pass@8": 0.9714815965888218, + "pass@16": 0.9781257545111052, + "pass@32": 0.9834450148745346, + "pass@64": 0.9875942130102727, + "pass@128": 0.9903572557698388 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9401963495575221, + "pass@2": 0.9506222708057512, + "pass@4": 0.9587029428290492, + "pass@8": 0.9658143010418404, + "pass@16": 0.9722315605765348, + "pass@32": 0.9776190501932904, + "pass@64": 0.9821069470948363, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998184250764526, + "pass@2": 0.9991895723470345, + "pass@4": 0.9997933970982985, + "pass@8": 0.9999877146197467, + "pass@16": 0.99999997197471, + "pass@32": 0.9999999999999853, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969482421875, + "pass@2": 0.997114911417323, + "pass@4": 0.9973405775449944, + "pass@8": 0.9977489343884937, + "pass@16": 0.9984154059245618, + "pass@32": 0.9992780160292464, + "pass@64": 0.999909940944882, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961309523809524, + "pass@2": 0.9987071928508937, + "pass@4": 0.999725366472048, + "pass@8": 0.9999813917311221, + "pass@16": 0.9999999349314345, + "pass@32": 0.9999999999998986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.9998817004845547, + "pass@4": 0.9999975042875557, + "pass@8": 0.9999999989853416, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9947916666666666, + "pass@2": 0.9978863819907128, + "pass@4": 0.9992973630700009, + "pass@8": 0.9998804787336676, + "pass@16": 0.9999966807249981, + "pass@32": 0.999999999134954, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9931117957746479, + "pass@2": 0.9946046356881446, + "pass@4": 0.9952563448935078, + "pass@8": 0.9960270148775638, + "pass@16": 0.9971824470231253, + "pass@32": 0.9985597501374238, + "pass@64": 0.9996474842017711, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.961003895184136, + "pass@2": 0.9758757193682941, + "pass@4": 0.9859773047136813, + "pass@8": 0.9925369319619592, + "pass@16": 0.996719964796309, + "pass@32": 0.9992331687805646, + "pass@64": 0.9999750061727128, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9469014830508474, + "pass@2": 0.9680802777036346, + "pass@4": 0.9823547948985191, + "pass@8": 0.9907254129910008, + "pass@16": 0.9957055694665387, + "pass@32": 0.998830769032777, + "pass@64": 0.9999113679009122, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9084541536050157, + "pass@2": 0.9421539845234863, + "pass@4": 0.9648193210762447, + "pass@8": 0.9782534132080241, + "pass@16": 0.9858754146175864, + "pass@32": 0.9896348514061588, + "pass@64": 0.9917705113956121, + "pass@128": 0.9937304075235109 + }, + "14": { + "pass@1": 0.911117601246106, + "pass@2": 0.9386605587852919, + "pass@4": 0.9593386028265156, + "pass@8": 0.973756274172, + "pass@16": 0.9835840404357657, + "pass@32": 0.9901129291802138, + "pass@64": 0.9944569067911632, + "pass@128": 0.9968847352024922 + }, + "15": { + "pass@1": 0.9607794436416185, + "pass@2": 0.9809912782759094, + "pass@4": 0.9878286179726087, + "pass@8": 0.9906328734957064, + "pass@16": 0.9918557494374052, + "pass@32": 0.992594484098195, + "pass@64": 0.9935027989512581, + "pass@128": 0.9942196531791907 + }, + "16": { + "pass@1": 0.8590725806451613, + "pass@2": 0.8959617887985772, + "pass@4": 0.92299481324915, + "pass@8": 0.9429119820308187, + "pass@16": 0.9569669200321872, + "pass@32": 0.9684234527512353, + "pass@64": 0.9777462994805372, + "pass@128": 0.9838709677419355 + }, + "17": { + "pass@1": 0.8935603323699421, + "pass@2": 0.9237261617586823, + "pass@4": 0.9405292829183923, + "pass@8": 0.9529428599398514, + "pass@16": 0.963253636094547, + "pass@32": 0.9715690776732758, + "pass@64": 0.9776152235402346, + "pass@128": 0.9826589595375722 + }, + "18": { + "pass@1": 0.8213169642857143, + "pass@2": 0.8683225534308209, + "pass@4": 0.8961326584176972, + "pass@8": 0.9165233758627166, + "pass@16": 0.9336095470655086, + "pass@32": 0.9487447437891168, + "pass@64": 0.9618053813005009, + "pass@128": 0.9714285714285714 + }, + "19": { + "pass@1": 0.7183511636636637, + "pass@2": 0.7900654098980873, + "pass@4": 0.8403002755286215, + "pass@8": 0.8751205968307463, + "pass@16": 0.8989661660960092, + "pass@32": 0.917543447331758, + "pass@64": 0.932127476264556, + "pass@128": 0.93993993993994 + }, + "20": { + "pass@1": 0.7155827702702703, + "pass@2": 0.8054037058712256, + "pass@4": 0.8563406066358814, + "pass@8": 0.8878686428984232, + "pass@16": 0.9121683780580442, + "pass@32": 0.9349939045842077, + "pass@64": 0.9550958412627336, + "pass@128": 0.9669669669669669 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 766124, + "answer_accuracy": 0.9330231878409977, + "avg_response_len": 291.9812658320343, + "resp_tokens_sum": 239751657, + "pass_at_k": { + "pass@1": 0.9330231878409977, + "pass@2": 0.953860331346917, + "pass@4": 0.9672898984908648, + "pass@8": 0.9764879321348401, + "pass@16": 0.9828421816715034, + "pass@32": 0.9873607429881609, + "pass@64": 0.9907464635833655, + "pass@128": 0.9936087295401403 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9255416027607362, + "pass@2": 0.9411788681706198, + "pass@4": 0.9550019118584102, + "pass@8": 0.964868285928447, + "pass@16": 0.9719946880736567, + "pass@32": 0.9785032582118173, + "pass@64": 0.9854509352402538, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9996178567167742, + "pass@4": 0.9999766702457646, + "pass@8": 0.99999989566327, + "pass@16": 0.9999999999992779, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9975317028985508, + "pass@2": 0.9996298356727149, + "pass@4": 0.9999874317340767, + "pass@8": 0.999999982420672, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993508237822349, + "pass@2": 0.9999333726282066, + "pass@4": 0.9999991442688574, + "pass@8": 0.9999999999122846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978775289017341, + "pass@2": 0.9996525960356833, + "pass@4": 0.9999823395449846, + "pass@8": 0.99999995023541, + "pass@16": 0.9999999999998741, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9940801056338028, + "pass@2": 0.9969987246312523, + "pass@4": 0.9984681511642031, + "pass@8": 0.9993523248812373, + "pass@16": 0.9998691472743508, + "pass@32": 0.9999964186630312, + "pass@64": 0.9999999998118273, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993123128742515, + "pass@2": 0.9944116442312227, + "pass@4": 0.9948953917249864, + "pass@8": 0.9956527038685988, + "pass@16": 0.9968535622066148, + "pass@32": 0.9983693531920645, + "pass@64": 0.9996149126751299, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9734563253012049, + "pass@2": 0.9823694383834549, + "pass@4": 0.987195932210281, + "pass@8": 0.9907554712427886, + "pass@16": 0.9940552664938294, + "pass@32": 0.9966879425459982, + "pass@64": 0.998400809093451, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9348795062695925, + "pass@2": 0.9590008145533532, + "pass@4": 0.976941083579286, + "pass@8": 0.9878423646843028, + "pass@16": 0.9936105268573175, + "pass@32": 0.9967195694500087, + "pass@64": 0.9983826288179706, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9399728641456583, + "pass@2": 0.961666556386334, + "pass@4": 0.9770743410224988, + "pass@8": 0.9880556844662887, + "pass@16": 0.9949278244154547, + "pass@32": 0.9980192365368127, + "pass@64": 0.999299221759988, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9399442616959064, + "pass@2": 0.9629545690012435, + "pass@4": 0.979254098720116, + "pass@8": 0.9904509185434833, + "pass@16": 0.9963750780080175, + "pass@32": 0.9990808958940353, + "pass@64": 0.9999511240349822, + "pass@128": 1.0 + }, + "15": { + "pass@1": 0.9578361742424243, + "pass@2": 0.9781145609639704, + "pass@4": 0.9891054819852062, + "pass@8": 0.9964601309728172, + "pass@16": 0.9994909800349003, + "pass@32": 0.9999845859908743, + "pass@64": 0.9999999969188734, + "pass@128": 1.0 + }, + "16": { + "pass@1": 0.8879763176638177, + "pass@2": 0.918719294703547, + "pass@4": 0.9389211383299313, + "pass@8": 0.9522550224776325, + "pass@16": 0.9622880180618023, + "pass@32": 0.9711609398649986, + "pass@64": 0.9784153797441794, + "pass@128": 0.9857549857549858 + }, + "17": { + "pass@1": 0.8806531444099379, + "pass@2": 0.9134255636523697, + "pass@4": 0.9319340411447012, + "pass@8": 0.9455538506849092, + "pass@16": 0.9575326946061652, + "pass@32": 0.9686242097805646, + "pass@64": 0.9789813555707505, + "pass@128": 0.9875776397515528 + }, + "18": { + "pass@1": 0.8600183823529411, + "pass@2": 0.9148224004168601, + "pass@4": 0.946246044795871, + "pass@8": 0.965380451817509, + "pass@16": 0.9765385504168345, + "pass@32": 0.9824988391462636, + "pass@64": 0.9867292851981442, + "pass@128": 0.9911764705882353 + }, + "19": { + "pass@1": 0.7423735119047619, + "pass@2": 0.8135465000468688, + "pass@4": 0.8603529316089945, + "pass@8": 0.8940199961295004, + "pass@16": 0.9191097672675497, + "pass@32": 0.9370573825963017, + "pass@64": 0.9482075383975361, + "pass@128": 0.9553571428571429 + }, + "20": { + "pass@1": 0.6969966317365269, + "pass@2": 0.7815660509689274, + "pass@4": 0.8387318512581132, + "pass@8": 0.8791036231156621, + "pass@16": 0.908555207050984, + "pass@32": 0.9310486628748871, + "pass@64": 0.9493486850175739, + "pass@128": 0.9670658682634731 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 743281, + "answer_accuracy": 0.9277652680140598, + "avg_response_len": 312.67436141955585, + "resp_tokens_sum": 250499690, + "pass_at_k": { + "pass@1": 0.9277652680140598, + "pass@2": 0.9507856324687619, + "pass@4": 0.9651909789086295, + "pass@8": 0.9746690687410497, + "pass@16": 0.9811944143836966, + "pass@32": 0.9857273357641817, + "pass@64": 0.9888025664674049, + "pass@128": 0.990893113915961 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9516091417910447, + "pass@2": 0.9654123575038193, + "pass@4": 0.9758070306510193, + "pass@8": 0.9831957697436913, + "pass@16": 0.9889855702187899, + "pass@32": 0.9935355804887811, + "pass@64": 0.9960768024790906, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999974891531417, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999086257309941, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920955882352941, + "pass@2": 0.9953993457619268, + "pass@4": 0.9967361543042413, + "pass@8": 0.9971991038919662, + "pass@16": 0.9974260398273237, + "pass@32": 0.997794117637879, + "pass@64": 0.9985294117647059, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9977636300209309, + "pass@4": 0.9992256535575459, + "pass@8": 0.9999093452430227, + "pass@16": 0.9999989464974799, + "pass@32": 0.999999999952471, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9961289414414415, + "pass@2": 0.998992102338559, + "pass@4": 0.9998528829729619, + "pass@8": 0.9999953036740797, + "pass@16": 0.9999999964703241, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9918860530546624, + "pass@2": 0.995059751373522, + "pass@4": 0.9964205448997332, + "pass@8": 0.9967600466956527, + "pass@16": 0.9967844556154133, + "pass@32": 0.996784565916054, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9640128968253968, + "pass@2": 0.9785726002999626, + "pass@4": 0.9885006380154859, + "pass@8": 0.9945696221853831, + "pass@16": 0.9979003162132001, + "pass@32": 0.9994098499839769, + "pass@64": 0.9999560277941457, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9435923165137615, + "pass@2": 0.9645033440487374, + "pass@4": 0.9802609306864158, + "pass@8": 0.9905943249666102, + "pass@16": 0.99650106594029, + "pass@32": 0.9991008676779993, + "pass@64": 0.9999110056404356, + "pass@128": 1.0 + }, + "13": { + "pass@1": 0.9024643132716049, + "pass@2": 0.9357103474044912, + "pass@4": 0.9593459379498855, + "pass@8": 0.9761529431305727, + "pass@16": 0.9876406570044788, + "pass@32": 0.993949453016883, + "pass@64": 0.9968475560089626, + "pass@128": 1.0 + }, + "14": { + "pass@1": 0.9529163575667656, + "pass@2": 0.9730652293277879, + "pass@4": 0.9829347865270549, + "pass@8": 0.9883460288220425, + "pass@16": 0.9917271270318293, + "pass@32": 0.9941231337871526, + "pass@64": 0.9955242663704985, + "pass@128": 0.9970326409495549 + }, + "15": { + "pass@1": 0.9280237268518519, + "pass@2": 0.9561343352046277, + "pass@4": 0.9718491169506588, + "pass@8": 0.9809465793474069, + "pass@16": 0.9868638704494458, + "pass@32": 0.9906400734779464, + "pass@64": 0.9928668799300133, + "pass@128": 0.9938271604938271 + }, + "16": { + "pass@1": 0.8850018436578171, + "pass@2": 0.9218477806424639, + "pass@4": 0.9422496073499659, + "pass@8": 0.9553439523514501, + "pass@16": 0.9642035935250982, + "pass@32": 0.9694637950507127, + "pass@64": 0.9723053624794668, + "pass@128": 0.9734513274336283 + }, + "17": { + "pass@1": 0.8449030496987951, + "pass@2": 0.8803859631439148, + "pass@4": 0.9044475689597533, + "pass@8": 0.9227436317910335, + "pass@16": 0.9365260593535809, + "pass@32": 0.9473040501395271, + "pass@64": 0.9558809363143925, + "pass@128": 0.9608433734939759 + }, + "18": { + "pass@1": 0.8615675403225806, + "pass@2": 0.9115205105410208, + "pass@4": 0.9396496183944743, + "pass@8": 0.9578697359344478, + "pass@16": 0.971502040708731, + "pass@32": 0.9817201040433834, + "pass@64": 0.9877409575782228, + "pass@128": 0.9903225806451613 + }, + "19": { + "pass@1": 0.7207090256797583, + "pass@2": 0.7996095701405903, + "pass@4": 0.8518439339719992, + "pass@8": 0.8891592506852346, + "pass@16": 0.9172980787103946, + "pass@32": 0.938093019556988, + "pass@64": 0.952800076816336, + "pass@128": 0.9637462235649547 + }, + "20": { + "pass@1": 0.6951717342342343, + "pass@2": 0.7902046978080438, + "pass@4": 0.8496138244543745, + "pass@8": 0.8865549890499133, + "pass@16": 0.9103071020752801, + "pass@32": 0.928051630813609, + "pass@64": 0.9431616248226694, + "pass@128": 0.954954954954955 + } + } + } + } + } +] \ No newline at end of file