diff --git a/.gitattributes b/.gitattributes index 492ec3bad2097b4d5f20024b2d8576640bc88002..05575841280774a468913deb807c9df21fc0a582 100644 --- a/.gitattributes +++ b/.gitattributes @@ -92,3 +92,106 @@ data/context/teachers_in_school/heldout/op6_50k.jsonl filter=lfs diff=lfs merge= data/context/teachers_in_school/heldout/op7_50k.jsonl filter=lfs diff=lfs merge=lfs -text data/context/teachers_in_school/heldout/op8_50k.jsonl filter=lfs diff=lfs merge=lfs -text data/context/teachers_in_school/heldout/op9_50k.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text +results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/graphs/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ac3cadd2312d6dad3620b19cf8aaaeaa4a623cb0 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e31eecdcea7aae730ad9cf8d0ab92c3308fedd3855b97e3e9ece59ccb9e73220 +size 1795735165 diff --git a/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json b/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..746dc933941e33dde6e44ef4873cf37a696173b8 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op11-14", + "id": { + "answer_accuracy": 0.9566944444444444, + "avg_response_len": 201.07631163194444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9566944444444444, + "pass@2": 0.9703619996719052, + "pass@4": 0.9787109671603543, + "pass@8": 0.9843092719873776, + "pass@16": 0.9881272957042198, + "pass@32": 0.9908108707272433, + "pass@64": 0.9929228603396479, + "pass@128": 0.9947777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9013359375, + "pass@2": 0.9259847440944887, + "pass@4": 0.9431592971503563, + "pass@8": 0.956216013428469, + "pass@16": 0.9655633252300617, + "pass@32": 0.9717840648944569, + "pass@64": 0.9764011519484884, + "pass@128": 0.981 + }, + "2": { + "pass@1": 0.9834765625, + "pass@2": 0.9886993110236222, + "pass@4": 0.9916434472253468, + "pass@8": 0.9931709501255696, + "pass@16": 0.9940232177790858, + "pass@32": 0.9945486103085506, + "pass@64": 0.9948778631860671, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.9914375, + "pass@2": 0.9955533956692916, + "pass@4": 0.9974521435133106, + "pass@8": 0.9985974908269989, + "pass@16": 0.9994160208486209, + "pass@32": 0.9998813614910528, + "pass@64": 0.99999788750106, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9889921875, + "pass@2": 0.9921766732283467, + "pass@4": 0.9943459424446944, + "pass@8": 0.9959193357486759, + "pass@16": 0.9970008520200708, + "pass@32": 0.9978515643805362, + "pass@64": 0.9987504617142802, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9543515625, + "pass@2": 0.9634336860236224, + "pass@4": 0.9701805443382078, + "pass@8": 0.9754922702366896, + "pass@16": 0.9795036618309861, + "pass@32": 0.9827616662620356, + "pass@64": 0.9859211476022041, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.9538828125, + "pass@2": 0.9683150836614177, + "pass@4": 0.9770699007311582, + "pass@8": 0.9835945887414447, + "pass@16": 0.988634619084515, + "pass@32": 0.9923047369821443, + "pass@64": 0.9946799358205762, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.9616796875, + "pass@2": 0.9749260580708665, + "pass@4": 0.9833017036932883, + "pass@8": 0.9889375836603118, + "pass@16": 0.992596462343972, + "pass@32": 0.9949202064490399, + "pass@64": 0.9966019468994491, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.940296875, + "pass@2": 0.9630469980314962, + "pass@4": 0.9771136480127484, + "pass@8": 0.9858646091378575, + "pass@16": 0.9910784862737938, + "pass@32": 0.9945541301929133, + "pass@64": 0.9975505786482517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.934796875, + "pass@2": 0.961122047244095, + "pass@4": 0.9741320773340832, + "pass@8": 0.9809906059803875, + "pass@16": 0.9853290159268713, + "pass@32": 0.9886914955844506, + "pass@64": 0.9915247697364523, + "pass@128": 0.994 + } + }, + "per_op_accuracy": { + "10": 0.9013359375, + "2": 0.9834765625, + "3": 0.9914375, + "4": 0.9889921875, + "5": 0.9543515625, + "6": 0.9538828125, + "7": 0.9616796875, + "8": 0.940296875, + "9": 0.934796875 + }, + "per_op_avg_response_len": { + "10": 279.94903125, + "2": 117.7111875, + "3": 145.45546875, + "4": 160.1310546875, + "5": 175.9252734375, + "6": 201.25878125, + "7": 225.22690625, + "8": 244.521140625, + "9": 259.5079609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.94903125, + "2": 117.7111875, + "3": 145.45546875, + "4": 160.1310546875, + "5": 175.9252734375, + "6": 201.25878125, + "7": 225.22690625, + "8": 244.521140625, + "9": 259.5079609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 360710, + "answer_accuracy": 0.9517213356973995, + "avg_response_len": 191.63888625042216, + "resp_tokens_sum": 72632671, + "pass_at_k": { + "pass@1": 0.9517213356973995, + "pass@2": 0.9669526016694736, + "pass@4": 0.9763565958130618, + "pass@8": 0.9828898231964195, + "pass@16": 0.987507739394257, + "pass@32": 0.9905947455484967, + "pass@64": 0.9928967227104302, + "pass@128": 0.9952718676122931 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9076327433628318, + "pass@2": 0.9269853436462037, + "pass@4": 0.9398492909625238, + "pass@8": 0.9506059829868715, + "pass@16": 0.9596484055654059, + "pass@32": 0.9658798721555774, + "pass@64": 0.9688322453458257, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9780957943925234, + "pass@2": 0.9850905757107465, + "pass@4": 0.9894686271692673, + "pass@8": 0.9920504616490534, + "pass@16": 0.9933079829828495, + "pass@32": 0.9937265331983628, + "pass@64": 0.9937693846777408, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.991829128440367, + "pass@2": 0.9972839010089335, + "pass@4": 0.9991199451559382, + "pass@8": 0.9998000718183061, + "pass@16": 0.9999896701598865, + "pass@32": 0.999999988244179, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9865234375, + "pass@2": 0.9907341904527559, + "pass@4": 0.9941022271630107, + "pass@8": 0.9962775829156062, + "pass@16": 0.997180906859181, + "pass@32": 0.9976553064004617, + "pass@64": 0.9984374999950079, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9469990079365079, + "pass@2": 0.9564476315460567, + "pass@4": 0.9630626984126983, + "pass@8": 0.9694378790828586, + "pass@16": 0.9759160456546958, + "pass@32": 0.9818998343552849, + "pass@64": 0.9875177250605657, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.926529752994012, + "pass@2": 0.9463030140506391, + "pass@4": 0.9610347533279894, + "pass@8": 0.9741423767959982, + "pass@16": 0.9846537870946782, + "pass@32": 0.9919116519382953, + "pass@64": 0.996595146234816, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9607063609467456, + "pass@2": 0.9766032678330153, + "pass@4": 0.9867770337879364, + "pass@8": 0.9932539965998115, + "pass@16": 0.9966691243987713, + "pass@32": 0.997758364804311, + "pass@64": 0.9985207077144822, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9363231169871795, + "pass@2": 0.9618858677821527, + "pass@4": 0.9778153304755176, + "pass@8": 0.9872242972247056, + "pass@16": 0.9919063397806129, + "pass@32": 0.9944326739175762, + "pass@64": 0.9972112783705197, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9345290492957746, + "pass@2": 0.9633556753909281, + "pass@4": 0.977406277032272, + "pass@8": 0.9842799230966626, + "pass@16": 0.9890320300163855, + "pass@32": 0.9926185701753283, + "pass@64": 0.9956966535148944, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 375668, + "answer_accuracy": 0.9616337647444299, + "avg_response_len": 198.80759542922672, + "resp_tokens_sum": 77665380, + "pass_at_k": { + "pass@1": 0.9616337647444299, + "pass@2": 0.9739375996506754, + "pass@4": 0.9813179619442245, + "pass@8": 0.9862698702130008, + "pass@16": 0.9896383230988169, + "pass@32": 0.9920310335769109, + "pass@64": 0.9939622377244645, + "pass@128": 0.9954128440366973 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8978144171779141, + "pass@2": 0.9213515500458915, + "pass@4": 0.9390322137646904, + "pass@8": 0.9526189121876671, + "pass@16": 0.9621797529616819, + "pass@32": 0.9684335022171842, + "pass@64": 0.9742357566082435, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.991950037993921, + "pass@2": 0.9956385521623626, + "pass@4": 0.996789173769692, + "pass@8": 0.9969556274713829, + "pass@16": 0.996960482749072, + "pass@32": 0.9969604863221885, + "pass@64": 0.9969604863221885, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9910748106060606, + "pass@2": 0.9938342579336673, + "pass@4": 0.9954610878185681, + "pass@8": 0.9967861359234476, + "pass@16": 0.9983705508051556, + "pass@32": 0.999643213120596, + "pass@64": 0.99999359856941, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9862934541420119, + "pass@2": 0.9914085431906072, + "pass@4": 0.9944976862174774, + "pass@8": 0.9968238286910404, + "pass@16": 0.9986471171735518, + "pass@32": 0.9997422779717037, + "pass@64": 0.9999955420471618, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9700860507246377, + "pass@2": 0.9768297814675341, + "pass@4": 0.9813448272770252, + "pass@8": 0.9845774240321683, + "pass@16": 0.9867515567319303, + "pass@32": 0.9884426560783856, + "pass@64": 0.9902316406247298, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9678545845272206, + "pass@2": 0.9800999199061439, + "pass@4": 0.9855557597742974, + "pass@8": 0.988306274970953, + "pass@16": 0.9904143515720015, + "pass@32": 0.9923533345945439, + "pass@64": 0.9938368819680918, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.965904985549133, + "pass@2": 0.9778386850848846, + "pass@4": 0.9852525098018816, + "pass@8": 0.9905732375817846, + "pass@16": 0.9943721417100441, + "pass@32": 0.997168595303731, + "pass@64": 0.9991059501069279, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9451804577464789, + "pass@2": 0.9669578989686152, + "pass@4": 0.9803207029579047, + "pass@8": 0.9881750952049374, + "pass@16": 0.9925613357660423, + "pass@32": 0.9958566064759846, + "pass@64": 0.9987507974877009, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9378508607784432, + "pass@2": 0.960500259323872, + "pass@4": 0.9725681423991157, + "pass@8": 0.9806556822439841, + "pass@16": 0.9856425911302683, + "pass@32": 0.988864619985313, + "pass@64": 0.9917644684832534, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 365734, + "answer_accuracy": 0.9565774606628724, + "avg_response_len": 212.74967567793772, + "resp_tokens_sum": 81341860, + "pass_at_k": { + "pass@1": 0.9565774606628724, + "pass@2": 0.9700883124774305, + "pass@4": 0.9783811196474563, + "pass@8": 0.9837131026152431, + "pass@16": 0.9871975503495145, + "pass@32": 0.989778399899347, + "pass@64": 0.9918867752180041, + "pass@128": 0.9936391027787077 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8983908582089553, + "pass@2": 0.9294809172640736, + "pass@4": 0.950525032318721, + "pass@8": 0.9653934920081807, + "pass@16": 0.9748415411280041, + "pass@32": 0.9810193030176554, + "pass@64": 0.9861676540954208, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9804464285714286, + "pass@2": 0.9854861501687289, + "pass@4": 0.9888010849536667, + "pass@8": 0.9906410014232532, + "pass@16": 0.9919181611918471, + "pass@32": 0.99303540900536, + "pass@64": 0.9939368305843209, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.991413083090379, + "pass@2": 0.9955575945226233, + "pass@4": 0.9977777331405003, + "pass@8": 0.9991937098766038, + "pass@16": 0.9998749765033144, + "pass@32": 0.9999973906863244, + "pass@64": 0.9999999999217336, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9939692982456141, + "pass@2": 0.9942855107749691, + "pass@4": 0.994424011143344, + "pass@8": 0.9946902196435972, + "pass@16": 0.9952053690657088, + "pass@32": 0.9961665975963532, + "pass@64": 0.9978127734033246, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9451976102941176, + "pass@2": 0.9563129631773971, + "pass@4": 0.9654465556695121, + "pass@8": 0.9718827265720327, + "pass@16": 0.9754730011095328, + "pass@32": 0.9777955943914058, + "pass@64": 0.9800680829193935, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9673205835962145, + "pass@2": 0.9785331385528704, + "pass@4": 0.9846225014459944, + "pass@8": 0.9883663877814471, + "pass@16": 0.9908695441522511, + "pass@32": 0.9926653989314137, + "pass@64": 0.9935901740419045, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9580943433544303, + "pass@2": 0.9699429383035981, + "pass@4": 0.9774484110174518, + "pass@8": 0.9825297234375893, + "pass@16": 0.9862960040047856, + "pass@32": 0.9894226112977601, + "pass@64": 0.9918078764397377, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9388138138138138, + "pass@2": 0.9599656250738929, + "pass@4": 0.9730372833463381, + "pass@8": 0.982127536354344, + "pass@16": 0.9887220242201132, + "pass@32": 0.9932794012962612, + "pass@64": 0.9965889690345809, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9318227491961415, + "pass@2": 0.9592401796338966, + "pass@4": 0.9720742425283658, + "pass@8": 0.9775956122558883, + "pass@16": 0.9807653370853542, + "pass@32": 0.9840228942029388, + "pass@64": 0.9865052259333055, + "pass@128": 0.9871382636655949 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.510975, + "avg_response_len": 362.007928125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.510975, + "pass@2": 0.5936387180118026, + "pass@4": 0.6617797073396955, + "pass@8": 0.7167137733109136, + "pass@16": 0.7611447510468944, + "pass@32": 0.7983214509893242, + "pass@64": 0.8300479591802541, + "pass@128": 0.8567 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.89196875, + "pass@2": 0.9299221210629928, + "pass@4": 0.9529178682977126, + "pass@8": 0.9672675175171029, + "pass@16": 0.9756364231803202, + "pass@32": 0.9807627127916235, + "pass@64": 0.9844788117192562, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.841203125, + "pass@2": 0.8876090059055125, + "pass@4": 0.9141195879265096, + "pass@8": 0.9306535756844154, + "pass@16": 0.9413871883991257, + "pass@32": 0.9492624080901529, + "pass@64": 0.9559608145833538, + "pass@128": 0.962 + }, + "13": { + "pass@1": 0.6938671875, + "pass@2": 0.7763882874015748, + "pass@4": 0.8326519359767524, + "pass@8": 0.8707372795349085, + "pass@16": 0.8970861149243851, + "pass@32": 0.9176561586813041, + "pass@64": 0.9349613423561156, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.62446875, + "pass@2": 0.7159948326771655, + "pass@4": 0.7863533314585672, + "pass@8": 0.8389012067186266, + "pass@16": 0.8768195969158379, + "pass@32": 0.9040171453745361, + "pass@64": 0.9236577047523269, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.5383828125, + "pass@2": 0.6369622293307086, + "pass@4": 0.7131546289838765, + "pass@8": 0.769510564426078, + "pass@16": 0.8106159119539578, + "pass@32": 0.8417999582459403, + "pass@64": 0.8667511771745826, + "pass@128": 0.886 + }, + "16": { + "pass@1": 0.4368359375, + "pass@2": 0.5438690944881892, + "pass@4": 0.6312509509748778, + "pass@8": 0.6990218214156836, + "pass@16": 0.7515598142013872, + "pass@32": 0.7941632145309236, + "pass@64": 0.8296922098460493, + "pass@128": 0.859 + }, + "17": { + "pass@1": 0.352109375, + "pass@2": 0.4523837352362206, + "pass@4": 0.5387510978627673, + "pass@8": 0.6097353386899184, + "pass@16": 0.6696671824626427, + "pass@32": 0.7233398987358384, + "pass@64": 0.7717775833444259, + "pass@128": 0.817 + }, + "18": { + "pass@1": 0.3154453125, + "pass@2": 0.41220497047244115, + "pass@4": 0.49942514810648647, + "pass@8": 0.572338038929963, + "pass@16": 0.6328966641461382, + "pass@32": 0.6842474572023839, + "pass@64": 0.7286842008450987, + "pass@128": 0.767 + }, + "19": { + "pass@1": 0.211578125, + "pass@2": 0.294009350393701, + "pass@4": 0.3782810793963256, + "pass@8": 0.45912572627706577, + "pass@16": 0.5336634516261636, + "pass@32": 0.5996903294576137, + "pass@64": 0.6561316526827659, + "pass@128": 0.704 + }, + "20": { + "pass@1": 0.203890625, + "pass@2": 0.2870435531496063, + "pass@4": 0.3708914444131982, + "pass@8": 0.44984666391540407, + "pass@16": 0.5221151626588344, + "pass@32": 0.588275226782918, + "pass@64": 0.6483840944984783, + "pass@128": 0.699 + } + }, + "per_op_accuracy": { + "11": 0.89196875, + "12": 0.841203125, + "13": 0.6938671875, + "14": 0.62446875, + "15": 0.5383828125, + "16": 0.4368359375, + "17": 0.352109375, + "18": 0.3154453125, + "19": 0.211578125, + "20": 0.203890625 + }, + "per_op_avg_response_len": { + "11": 298.1007421875, + "12": 308.0588671875, + "13": 326.875390625, + "14": 349.399546875, + "15": 365.40840625, + "16": 372.7167734375, + "17": 389.5476953125, + "18": 398.404875, + "19": 399.2373515625, + "20": 412.3296328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.1007421875, + "12": 308.0588671875, + "13": 326.875390625, + "14": 349.399546875, + "15": 365.40840625, + "16": 372.7167734375, + "17": 389.5476953125, + "18": 398.404875, + "19": 399.2373515625, + "20": 412.3296328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 223546, + "answer_accuracy": 0.5190053863298663, + "avg_response_len": 343.9361325222883, + "resp_tokens_sum": 148140171, + "pass_at_k": { + "pass@1": 0.5190053863298663, + "pass@2": 0.6017590761778867, + "pass@4": 0.6688807888334177, + "pass@8": 0.7220798095471659, + "pass@16": 0.7645909661017662, + "pass@32": 0.8002128845619237, + "pass@64": 0.8311640113734907, + "pass@128": 0.8564635958395245 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8755975566572238, + "pass@2": 0.9161817436595214, + "pass@4": 0.9432045066604631, + "pass@8": 0.9603687852296585, + "pass@16": 0.9706278871438179, + "pass@32": 0.9775053297819941, + "pass@64": 0.981852619948499, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8450300141242938, + "pass@2": 0.8938623965701327, + "pass@4": 0.9229093023859299, + "pass@8": 0.9409758597723392, + "pass@16": 0.9516659432562773, + "pass@32": 0.9588886011978869, + "pass@64": 0.965495662578892, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7161295062695925, + "pass@2": 0.7977161651815466, + "pass@4": 0.8520592728612683, + "pass@8": 0.8873050572913995, + "pass@16": 0.9106471139096689, + "pass@32": 0.9292828710771102, + "pass@64": 0.9443766484455557, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.6268983644859814, + "pass@2": 0.7150230272033751, + "pass@4": 0.7842801770105835, + "pass@8": 0.837397655361588, + "pass@16": 0.8762786423304784, + "pass@32": 0.9046818450843452, + "pass@64": 0.9289289086935143, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.5480717124277457, + "pass@2": 0.6457133835510445, + "pass@4": 0.7187044204373292, + "pass@8": 0.7727291108200902, + "pass@16": 0.8139621475751042, + "pass@32": 0.8458336523771858, + "pass@64": 0.8698378477573592, + "pass@128": 0.8872832369942196 + }, + "16": { + "pass@1": 0.4441280241935484, + "pass@2": 0.5569766795783586, + "pass@4": 0.6460751194407147, + "pass@8": 0.7137075598181689, + "pass@16": 0.7681949863235685, + "pass@32": 0.8142011454810046, + "pass@64": 0.8517005897273575, + "pass@128": 0.8774193548387097 + }, + "17": { + "pass@1": 0.36806809971098264, + "pass@2": 0.4666509973373991, + "pass@4": 0.5472581679638306, + "pass@8": 0.6127021215261312, + "pass@16": 0.6691159679443502, + "pass@32": 0.7203100001054069, + "pass@64": 0.7654746436146028, + "pass@128": 0.8063583815028902 + }, + "18": { + "pass@1": 0.304375, + "pass@2": 0.3967902840269967, + "pass@4": 0.47884594809577347, + "pass@8": 0.5467893711314302, + "pass@16": 0.6037676808016317, + "pass@32": 0.6536113740422012, + "pass@64": 0.6999131573501903, + "pass@128": 0.74 + }, + "19": { + "pass@1": 0.21579391891891891, + "pass@2": 0.3006612665815421, + "pass@4": 0.38616035720760156, + "pass@8": 0.46550902965231605, + "pass@16": 0.5350209172548729, + "pass@32": 0.595347618281271, + "pass@64": 0.6482623365836448, + "pass@128": 0.6936936936936937 + }, + "20": { + "pass@1": 0.22670326576576577, + "pass@2": 0.3139590131470051, + "pass@4": 0.39960391071611545, + "pass@8": 0.4769628415256585, + "pass@16": 0.5428805161447351, + "pass@32": 0.6007913078306004, + "pass@64": 0.6557323133549934, + "pass@128": 0.7027027027027027 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 218386, + "answer_accuracy": 0.50732697740113, + "avg_response_len": 357.6412940454951, + "resp_tokens_sum": 153951702, + "pass_at_k": { + "pass@1": 0.50732697740113, + "pass@2": 0.5890962047033392, + "pass@4": 0.6570096634051469, + "pass@8": 0.7120303717421669, + "pass@16": 0.7565837178460383, + "pass@32": 0.794083787266784, + "pass@64": 0.8263549990088304, + "pass@128": 0.8542967588462682 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9046733810240963, + "pass@2": 0.9394221818375864, + "pass@4": 0.9586932646220422, + "pass@8": 0.9705318114541525, + "pass@16": 0.9775325668360785, + "pass@32": 0.9814679960471665, + "pass@64": 0.9848638566163492, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8320190047021944, + "pass@2": 0.8726735862562636, + "pass@4": 0.8965225433223666, + "pass@8": 0.9119176475126256, + "pass@16": 0.9216786817337147, + "pass@32": 0.9290268632604375, + "pass@64": 0.9362206325610609, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.6824010854341737, + "pass@2": 0.7620663915172362, + "pass@4": 0.816710596574587, + "pass@8": 0.8568656338449239, + "pass@16": 0.8863689471035519, + "pass@32": 0.9093439500882927, + "pass@64": 0.929124181128099, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6183982090643275, + "pass@2": 0.7113501231753926, + "pass@4": 0.7827317758525795, + "pass@8": 0.8349723554689547, + "pass@16": 0.8735588597859756, + "pass@32": 0.9025529986048424, + "pass@64": 0.9217735735634356, + "pass@128": 0.9327485380116959 + }, + "15": { + "pass@1": 0.5301846590909091, + "pass@2": 0.6305125566690525, + "pass@4": 0.7065275675767798, + "pass@8": 0.7606926843299037, + "pass@16": 0.7990498123628644, + "pass@32": 0.8293580860368167, + "pass@64": 0.8559740595796936, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.4334045584045584, + "pass@2": 0.5336198459968144, + "pass@4": 0.6156393698650913, + "pass@8": 0.6793719515351224, + "pass@16": 0.7282840734236551, + "pass@32": 0.7671109052665265, + "pass@64": 0.800987680746752, + "pass@128": 0.8319088319088319 + }, + "17": { + "pass@1": 0.3566818711180124, + "pass@2": 0.46245017606494854, + "pass@4": 0.5551400752700942, + "pass@8": 0.6291890200102845, + "pass@16": 0.6878346867297733, + "pass@32": 0.7384604652449943, + "pass@64": 0.7881347414072649, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.32173713235294116, + "pass@2": 0.42347802802223233, + "pass@4": 0.5183494841637442, + "pass@8": 0.5990558759706359, + "pass@16": 0.6651894072773802, + "pass@32": 0.7191764944894445, + "pass@64": 0.7606179403198643, + "pass@128": 0.7911764705882353 + }, + "19": { + "pass@1": 0.20986793154761904, + "pass@2": 0.2867501669713159, + "pass@4": 0.3652424758289142, + "pass@8": 0.4410855714382008, + "pass@16": 0.5130674911401856, + "pass@32": 0.5803165592539755, + "pass@64": 0.6393250538858533, + "pass@128": 0.6934523809523809 + }, + "20": { + "pass@1": 0.18988585329341318, + "pass@2": 0.2721880304351926, + "pass@4": 0.35637058588110615, + "pass@8": 0.4371482552763727, + "pass@16": 0.512849984475913, + "pass@32": 0.5830933497627412, + "pass@64": 0.645622648235715, + "pass@128": 0.6976047904191617 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 212116, + "answer_accuracy": 0.5064658465770171, + "avg_response_len": 385.08145581830684, + "resp_tokens_sum": 161278275, + "pass_at_k": { + "pass@1": 0.5064658465770171, + "pass@2": 0.589956403533684, + "pass@4": 0.6593794990651408, + "pass@8": 0.7160088734150627, + "pass@16": 0.7622884677323257, + "pass@32": 0.8007317777396525, + "pass@64": 0.8326958532591422, + "pass@128": 0.8594132029339854 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8969246031746032, + "pass@2": 0.9353073053368325, + "pass@4": 0.9577159161652413, + "pass@8": 0.9715580156452523, + "pass@16": 0.9792506883459516, + "pass@32": 0.9836697037806984, + "pass@64": 0.9870159888279616, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8460196865443425, + "pass@2": 0.8954093073514888, + "pass@4": 0.921770653094968, + "pass@8": 0.9377565497507031, + "pass@16": 0.9494860703772144, + "pass@32": 0.9585818773272825, + "pass@64": 0.9648959273499919, + "pass@128": 0.9694189602446484 + }, + "13": { + "pass@1": 0.6845823688271605, + "pass@2": 0.7711701510401477, + "pass@4": 0.831109089373087, + "pass@8": 0.8697096758528202, + "pass@16": 0.8955431218247937, + "pass@32": 0.9153676932906923, + "pass@64": 0.9321230211088019, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.6283150964391692, + "pass@2": 0.721634121124325, + "pass@4": 0.7920033450937324, + "pass@8": 0.844320515659269, + "pass@16": 0.8806439842164706, + "pass@32": 0.9048698741234554, + "pass@64": 0.9205488513442519, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.5363859953703703, + "pass@2": 0.6341859719791972, + "pass@4": 0.713977784605628, + "pass@8": 0.7750546489304276, + "pass@16": 0.8188227494235403, + "pass@32": 0.8501646485533466, + "pass@64": 0.8744316116951758, + "pass@128": 0.8919753086419753 + }, + "16": { + "pass@1": 0.43372050147492625, + "pass@2": 0.5424948609852972, + "pass@4": 0.6338591301640388, + "pass@8": 0.7059378256142278, + "pass@16": 0.760447370706132, + "pass@32": 0.8038493559978188, + "pass@64": 0.8392871713523845, + "pass@128": 0.8702064896755162 + }, + "17": { + "pass@1": 0.331042921686747, + "pass@2": 0.42775160682098473, + "pass@4": 0.5139899623804555, + "pass@8": 0.5877757234896518, + "pass@16": 0.6526213506834679, + "pass@32": 0.7118324364171099, + "pass@64": 0.7624818371103406, + "pass@128": 0.8072289156626506 + }, + "18": { + "pass@1": 0.32104334677419355, + "pass@2": 0.41724465011430023, + "pass@4": 0.5019040053461057, + "pass@8": 0.5718798748514401, + "pass@16": 0.6303663786814787, + "pass@32": 0.6805273166490414, + "pass@64": 0.7261435356896053, + "pass@128": 0.7709677419354839 + }, + "19": { + "pass@1": 0.20907288519637462, + "pass@2": 0.2946860801198942, + "pass@4": 0.3835897539809034, + "pass@8": 0.4710165178235919, + "pass@16": 0.5532048615020811, + "pass@32": 0.623725705923306, + "pass@64": 0.6811089924313154, + "pass@128": 0.7250755287009063 + }, + "20": { + "pass@1": 0.19512481231231232, + "pass@2": 0.2750282270459435, + "pass@4": 0.35674344282415144, + "pass@8": 0.435467028303459, + "pass@16": 0.510642810713764, + "pass@32": 0.5809565839476649, + "pass@64": 0.6438056145361447, + "pass@128": 0.6966966966966966 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7221052631578947, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7221052631578947, + "pass@2": 0.7720865882718362, + "pass@4": 0.8119050409389371, + "pass@8": 0.8434695358418849, + "pass@16": 0.8686627985161548, + "pass@32": 0.8895006498125616, + "pass@64": 0.9071992281504938, + "pass@128": 0.9221052631578948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9013359375, + "pass@2": 0.9259847440944887, + "pass@4": 0.9431592971503563, + "pass@8": 0.956216013428469, + "pass@16": 0.9655633252300617, + "pass@32": 0.9717840648944569, + "pass@64": 0.9764011519484884, + "pass@128": 0.981 + }, + "2": { + "pass@1": 0.9834765625, + "pass@2": 0.9886993110236222, + "pass@4": 0.9916434472253468, + "pass@8": 0.9931709501255696, + "pass@16": 0.9940232177790858, + "pass@32": 0.9945486103085506, + "pass@64": 0.9948778631860671, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.9914375, + "pass@2": 0.9955533956692916, + "pass@4": 0.9974521435133106, + "pass@8": 0.9985974908269989, + "pass@16": 0.9994160208486209, + "pass@32": 0.9998813614910528, + "pass@64": 0.99999788750106, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9889921875, + "pass@2": 0.9921766732283467, + "pass@4": 0.9943459424446944, + "pass@8": 0.9959193357486759, + "pass@16": 0.9970008520200708, + "pass@32": 0.9978515643805362, + "pass@64": 0.9987504617142802, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9543515625, + "pass@2": 0.9634336860236224, + "pass@4": 0.9701805443382078, + "pass@8": 0.9754922702366896, + "pass@16": 0.9795036618309861, + "pass@32": 0.9827616662620356, + "pass@64": 0.9859211476022041, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.9538828125, + "pass@2": 0.9683150836614177, + "pass@4": 0.9770699007311582, + "pass@8": 0.9835945887414447, + "pass@16": 0.988634619084515, + "pass@32": 0.9923047369821443, + "pass@64": 0.9946799358205762, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.9616796875, + "pass@2": 0.9749260580708665, + "pass@4": 0.9833017036932883, + "pass@8": 0.9889375836603118, + "pass@16": 0.992596462343972, + "pass@32": 0.9949202064490399, + "pass@64": 0.9966019468994491, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.940296875, + "pass@2": 0.9630469980314962, + "pass@4": 0.9771136480127484, + "pass@8": 0.9858646091378575, + "pass@16": 0.9910784862737938, + "pass@32": 0.9945541301929133, + "pass@64": 0.9975505786482517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.934796875, + "pass@2": 0.961122047244095, + "pass@4": 0.9741320773340832, + "pass@8": 0.9809906059803875, + "pass@16": 0.9853290159268713, + "pass@32": 0.9886914955844506, + "pass@64": 0.9915247697364523, + "pass@128": 0.994 + }, + "11": { + "pass@1": 0.89196875, + "pass@2": 0.9299221210629928, + "pass@4": 0.9529178682977126, + "pass@8": 0.9672675175171029, + "pass@16": 0.9756364231803202, + "pass@32": 0.9807627127916235, + "pass@64": 0.9844788117192562, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.841203125, + "pass@2": 0.8876090059055125, + "pass@4": 0.9141195879265096, + "pass@8": 0.9306535756844154, + "pass@16": 0.9413871883991257, + "pass@32": 0.9492624080901529, + "pass@64": 0.9559608145833538, + "pass@128": 0.962 + }, + "13": { + "pass@1": 0.6938671875, + "pass@2": 0.7763882874015748, + "pass@4": 0.8326519359767524, + "pass@8": 0.8707372795349085, + "pass@16": 0.8970861149243851, + "pass@32": 0.9176561586813041, + "pass@64": 0.9349613423561156, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.62446875, + "pass@2": 0.7159948326771655, + "pass@4": 0.7863533314585672, + "pass@8": 0.8389012067186266, + "pass@16": 0.8768195969158379, + "pass@32": 0.9040171453745361, + "pass@64": 0.9236577047523269, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.5383828125, + "pass@2": 0.6369622293307086, + "pass@4": 0.7131546289838765, + "pass@8": 0.769510564426078, + "pass@16": 0.8106159119539578, + "pass@32": 0.8417999582459403, + "pass@64": 0.8667511771745826, + "pass@128": 0.886 + }, + "16": { + "pass@1": 0.4368359375, + "pass@2": 0.5438690944881892, + "pass@4": 0.6312509509748778, + "pass@8": 0.6990218214156836, + "pass@16": 0.7515598142013872, + "pass@32": 0.7941632145309236, + "pass@64": 0.8296922098460493, + "pass@128": 0.859 + }, + "17": { + "pass@1": 0.352109375, + "pass@2": 0.4523837352362206, + "pass@4": 0.5387510978627673, + "pass@8": 0.6097353386899184, + "pass@16": 0.6696671824626427, + "pass@32": 0.7233398987358384, + "pass@64": 0.7717775833444259, + "pass@128": 0.817 + }, + "18": { + "pass@1": 0.3154453125, + "pass@2": 0.41220497047244115, + "pass@4": 0.49942514810648647, + "pass@8": 0.572338038929963, + "pass@16": 0.6328966641461382, + "pass@32": 0.6842474572023839, + "pass@64": 0.7286842008450987, + "pass@128": 0.767 + }, + "19": { + "pass@1": 0.211578125, + "pass@2": 0.294009350393701, + "pass@4": 0.3782810793963256, + "pass@8": 0.45912572627706577, + "pass@16": 0.5336634516261636, + "pass@32": 0.5996903294576137, + "pass@64": 0.6561316526827659, + "pass@128": 0.704 + }, + "20": { + "pass@1": 0.203890625, + "pass@2": 0.2870435531496063, + "pass@4": 0.3708914444131982, + "pass@8": 0.44984666391540407, + "pass@16": 0.5221151626588344, + "pass@32": 0.588275226782918, + "pass@64": 0.6483840944984783, + "pass@128": 0.699 + } + }, + "per_op_accuracy": { + "10": 0.9013359375, + "2": 0.9834765625, + "3": 0.9914375, + "4": 0.9889921875, + "5": 0.9543515625, + "6": 0.9538828125, + "7": 0.9616796875, + "8": 0.940296875, + "9": 0.934796875, + "11": 0.89196875, + "12": 0.841203125, + "13": 0.6938671875, + "14": 0.62446875, + "15": 0.5383828125, + "16": 0.4368359375, + "17": 0.352109375, + "18": 0.3154453125, + "19": 0.211578125, + "20": 0.203890625 + }, + "per_op_avg_response_len": { + "10": 279.94903125, + "2": 117.7111875, + "3": 145.45546875, + "4": 160.1310546875, + "5": 175.9252734375, + "6": 201.25878125, + "7": 225.22690625, + "8": 244.521140625, + "9": 259.5079609375, + "11": 298.1007421875, + "12": 308.0588671875, + "13": 326.875390625, + "14": 349.399546875, + "15": 365.40840625, + "16": 372.7167734375, + "17": 389.5476953125, + "18": 398.404875, + "19": 399.2373515625, + "20": 412.3296328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.94903125, + "2": 117.7111875, + "3": 145.45546875, + "4": 160.1310546875, + "5": 175.9252734375, + "6": 201.25878125, + "7": 225.22690625, + "8": 244.521140625, + "9": 259.5079609375, + "11": 298.1007421875, + "12": 308.0588671875, + "13": 326.875390625, + "14": 349.399546875, + "15": 365.40840625, + "16": 372.7167734375, + "17": 389.5476953125, + "18": 398.404875, + "19": 399.2373515625, + "20": 412.3296328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 584256, + "answer_accuracy": 0.7215460006323111, + "avg_response_len": 272.650620949257, + "resp_tokens_sum": 220772842, + "pass_at_k": { + "pass@1": 0.7215460006323111, + "pass@2": 0.7726945850271698, + "pass@4": 0.8128004638992861, + "pass@8": 0.844156706546122, + "pass@16": 0.8689312388996039, + "pass@32": 0.8893245966044871, + "pass@64": 0.9068659649411043, + "pass@128": 0.9214353461903256 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9076327433628318, + "pass@2": 0.9269853436462037, + "pass@4": 0.9398492909625238, + "pass@8": 0.9506059829868715, + "pass@16": 0.9596484055654059, + "pass@32": 0.9658798721555774, + "pass@64": 0.9688322453458257, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9780957943925234, + "pass@2": 0.9850905757107465, + "pass@4": 0.9894686271692673, + "pass@8": 0.9920504616490534, + "pass@16": 0.9933079829828495, + "pass@32": 0.9937265331983628, + "pass@64": 0.9937693846777408, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.991829128440367, + "pass@2": 0.9972839010089335, + "pass@4": 0.9991199451559382, + "pass@8": 0.9998000718183061, + "pass@16": 0.9999896701598865, + "pass@32": 0.999999988244179, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9865234375, + "pass@2": 0.9907341904527559, + "pass@4": 0.9941022271630107, + "pass@8": 0.9962775829156062, + "pass@16": 0.997180906859181, + "pass@32": 0.9976553064004617, + "pass@64": 0.9984374999950079, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9469990079365079, + "pass@2": 0.9564476315460567, + "pass@4": 0.9630626984126983, + "pass@8": 0.9694378790828586, + "pass@16": 0.9759160456546958, + "pass@32": 0.9818998343552849, + "pass@64": 0.9875177250605657, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.926529752994012, + "pass@2": 0.9463030140506391, + "pass@4": 0.9610347533279894, + "pass@8": 0.9741423767959982, + "pass@16": 0.9846537870946782, + "pass@32": 0.9919116519382953, + "pass@64": 0.996595146234816, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9607063609467456, + "pass@2": 0.9766032678330153, + "pass@4": 0.9867770337879364, + "pass@8": 0.9932539965998115, + "pass@16": 0.9966691243987713, + "pass@32": 0.997758364804311, + "pass@64": 0.9985207077144822, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9363231169871795, + "pass@2": 0.9618858677821527, + "pass@4": 0.9778153304755176, + "pass@8": 0.9872242972247056, + "pass@16": 0.9919063397806129, + "pass@32": 0.9944326739175762, + "pass@64": 0.9972112783705197, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9345290492957746, + "pass@2": 0.9633556753909281, + "pass@4": 0.977406277032272, + "pass@8": 0.9842799230966626, + "pass@16": 0.9890320300163855, + "pass@32": 0.9926185701753283, + "pass@64": 0.9956966535148944, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8755975566572238, + "pass@2": 0.9161817436595214, + "pass@4": 0.9432045066604631, + "pass@8": 0.9603687852296585, + "pass@16": 0.9706278871438179, + "pass@32": 0.9775053297819941, + "pass@64": 0.981852619948499, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8450300141242938, + "pass@2": 0.8938623965701327, + "pass@4": 0.9229093023859299, + "pass@8": 0.9409758597723392, + "pass@16": 0.9516659432562773, + "pass@32": 0.9588886011978869, + "pass@64": 0.965495662578892, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7161295062695925, + "pass@2": 0.7977161651815466, + "pass@4": 0.8520592728612683, + "pass@8": 0.8873050572913995, + "pass@16": 0.9106471139096689, + "pass@32": 0.9292828710771102, + "pass@64": 0.9443766484455557, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.6268983644859814, + "pass@2": 0.7150230272033751, + "pass@4": 0.7842801770105835, + "pass@8": 0.837397655361588, + "pass@16": 0.8762786423304784, + "pass@32": 0.9046818450843452, + "pass@64": 0.9289289086935143, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.5480717124277457, + "pass@2": 0.6457133835510445, + "pass@4": 0.7187044204373292, + "pass@8": 0.7727291108200902, + "pass@16": 0.8139621475751042, + "pass@32": 0.8458336523771858, + "pass@64": 0.8698378477573592, + "pass@128": 0.8872832369942196 + }, + "16": { + "pass@1": 0.4441280241935484, + "pass@2": 0.5569766795783586, + "pass@4": 0.6460751194407147, + "pass@8": 0.7137075598181689, + "pass@16": 0.7681949863235685, + "pass@32": 0.8142011454810046, + "pass@64": 0.8517005897273575, + "pass@128": 0.8774193548387097 + }, + "17": { + "pass@1": 0.36806809971098264, + "pass@2": 0.4666509973373991, + "pass@4": 0.5472581679638306, + "pass@8": 0.6127021215261312, + "pass@16": 0.6691159679443502, + "pass@32": 0.7203100001054069, + "pass@64": 0.7654746436146028, + "pass@128": 0.8063583815028902 + }, + "18": { + "pass@1": 0.304375, + "pass@2": 0.3967902840269967, + "pass@4": 0.47884594809577347, + "pass@8": 0.5467893711314302, + "pass@16": 0.6037676808016317, + "pass@32": 0.6536113740422012, + "pass@64": 0.6999131573501903, + "pass@128": 0.74 + }, + "19": { + "pass@1": 0.21579391891891891, + "pass@2": 0.3006612665815421, + "pass@4": 0.38616035720760156, + "pass@8": 0.46550902965231605, + "pass@16": 0.5350209172548729, + "pass@32": 0.595347618281271, + "pass@64": 0.6482623365836448, + "pass@128": 0.6936936936936937 + }, + "20": { + "pass@1": 0.22670326576576577, + "pass@2": 0.3139590131470051, + "pass@4": 0.39960391071611545, + "pass@8": 0.4769628415256585, + "pass@16": 0.5428805161447351, + "pass@32": 0.6007913078306004, + "pass@64": 0.6557323133549934, + "pass@128": 0.7027027027027027 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 594054, + "answer_accuracy": 0.7234679462197974, + "avg_response_len": 282.07458349571317, + "resp_tokens_sum": 231617082, + "pass_at_k": { + "pass@1": 0.7234679462197974, + "pass@2": 0.7721883227671293, + "pass@4": 0.8113025592962161, + "pass@8": 0.8425025384347558, + "pass@16": 0.8674617623092497, + "pass@32": 0.8882591568285177, + "pass@64": 0.9060958084492297, + "pass@128": 0.9214341387373344 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8978144171779141, + "pass@2": 0.9213515500458915, + "pass@4": 0.9390322137646904, + "pass@8": 0.9526189121876671, + "pass@16": 0.9621797529616819, + "pass@32": 0.9684335022171842, + "pass@64": 0.9742357566082435, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.991950037993921, + "pass@2": 0.9956385521623626, + "pass@4": 0.996789173769692, + "pass@8": 0.9969556274713829, + "pass@16": 0.996960482749072, + "pass@32": 0.9969604863221885, + "pass@64": 0.9969604863221885, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9910748106060606, + "pass@2": 0.9938342579336673, + "pass@4": 0.9954610878185681, + "pass@8": 0.9967861359234476, + "pass@16": 0.9983705508051556, + "pass@32": 0.999643213120596, + "pass@64": 0.99999359856941, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9862934541420119, + "pass@2": 0.9914085431906072, + "pass@4": 0.9944976862174774, + "pass@8": 0.9968238286910404, + "pass@16": 0.9986471171735518, + "pass@32": 0.9997422779717037, + "pass@64": 0.9999955420471618, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9700860507246377, + "pass@2": 0.9768297814675341, + "pass@4": 0.9813448272770252, + "pass@8": 0.9845774240321683, + "pass@16": 0.9867515567319303, + "pass@32": 0.9884426560783856, + "pass@64": 0.9902316406247298, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9678545845272206, + "pass@2": 0.9800999199061439, + "pass@4": 0.9855557597742974, + "pass@8": 0.988306274970953, + "pass@16": 0.9904143515720015, + "pass@32": 0.9923533345945439, + "pass@64": 0.9938368819680918, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.965904985549133, + "pass@2": 0.9778386850848846, + "pass@4": 0.9852525098018816, + "pass@8": 0.9905732375817846, + "pass@16": 0.9943721417100441, + "pass@32": 0.997168595303731, + "pass@64": 0.9991059501069279, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9451804577464789, + "pass@2": 0.9669578989686152, + "pass@4": 0.9803207029579047, + "pass@8": 0.9881750952049374, + "pass@16": 0.9925613357660423, + "pass@32": 0.9958566064759846, + "pass@64": 0.9987507974877009, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9378508607784432, + "pass@2": 0.960500259323872, + "pass@4": 0.9725681423991157, + "pass@8": 0.9806556822439841, + "pass@16": 0.9856425911302683, + "pass@32": 0.988864619985313, + "pass@64": 0.9917644684832534, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9046733810240963, + "pass@2": 0.9394221818375864, + "pass@4": 0.9586932646220422, + "pass@8": 0.9705318114541525, + "pass@16": 0.9775325668360785, + "pass@32": 0.9814679960471665, + "pass@64": 0.9848638566163492, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8320190047021944, + "pass@2": 0.8726735862562636, + "pass@4": 0.8965225433223666, + "pass@8": 0.9119176475126256, + "pass@16": 0.9216786817337147, + "pass@32": 0.9290268632604375, + "pass@64": 0.9362206325610609, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.6824010854341737, + "pass@2": 0.7620663915172362, + "pass@4": 0.816710596574587, + "pass@8": 0.8568656338449239, + "pass@16": 0.8863689471035519, + "pass@32": 0.9093439500882927, + "pass@64": 0.929124181128099, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6183982090643275, + "pass@2": 0.7113501231753926, + "pass@4": 0.7827317758525795, + "pass@8": 0.8349723554689547, + "pass@16": 0.8735588597859756, + "pass@32": 0.9025529986048424, + "pass@64": 0.9217735735634356, + "pass@128": 0.9327485380116959 + }, + "15": { + "pass@1": 0.5301846590909091, + "pass@2": 0.6305125566690525, + "pass@4": 0.7065275675767798, + "pass@8": 0.7606926843299037, + "pass@16": 0.7990498123628644, + "pass@32": 0.8293580860368167, + "pass@64": 0.8559740595796936, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.4334045584045584, + "pass@2": 0.5336198459968144, + "pass@4": 0.6156393698650913, + "pass@8": 0.6793719515351224, + "pass@16": 0.7282840734236551, + "pass@32": 0.7671109052665265, + "pass@64": 0.800987680746752, + "pass@128": 0.8319088319088319 + }, + "17": { + "pass@1": 0.3566818711180124, + "pass@2": 0.46245017606494854, + "pass@4": 0.5551400752700942, + "pass@8": 0.6291890200102845, + "pass@16": 0.6878346867297733, + "pass@32": 0.7384604652449943, + "pass@64": 0.7881347414072649, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.32173713235294116, + "pass@2": 0.42347802802223233, + "pass@4": 0.5183494841637442, + "pass@8": 0.5990558759706359, + "pass@16": 0.6651894072773802, + "pass@32": 0.7191764944894445, + "pass@64": 0.7606179403198643, + "pass@128": 0.7911764705882353 + }, + "19": { + "pass@1": 0.20986793154761904, + "pass@2": 0.2867501669713159, + "pass@4": 0.3652424758289142, + "pass@8": 0.4410855714382008, + "pass@16": 0.5130674911401856, + "pass@32": 0.5803165592539755, + "pass@64": 0.6393250538858533, + "pass@128": 0.6934523809523809 + }, + "20": { + "pass@1": 0.18988585329341318, + "pass@2": 0.2721880304351926, + "pass@4": 0.35637058588110615, + "pass@8": 0.4371482552763727, + "pass@16": 0.512849984475913, + "pass@32": 0.5830933497627412, + "pass@64": 0.645622648235715, + "pass@128": 0.6976047904191617 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 577850, + "answer_accuracy": 0.7212738656334878, + "avg_response_len": 302.8390804741173, + "resp_tokens_sum": 242620135, + "pass_at_k": { + "pass@1": 0.7212738656334878, + "pass@2": 0.7713678130264008, + "pass@4": 0.8116175308081232, + "pass@8": 0.8437661082162949, + "pass@16": 0.8696224555542752, + "pass@32": 0.8909510236880488, + "pass@64": 0.9086669802588497, + "pass@128": 0.9234702029078128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8983908582089553, + "pass@2": 0.9294809172640736, + "pass@4": 0.950525032318721, + "pass@8": 0.9653934920081807, + "pass@16": 0.9748415411280041, + "pass@32": 0.9810193030176554, + "pass@64": 0.9861676540954208, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9804464285714286, + "pass@2": 0.9854861501687289, + "pass@4": 0.9888010849536667, + "pass@8": 0.9906410014232532, + "pass@16": 0.9919181611918471, + "pass@32": 0.99303540900536, + "pass@64": 0.9939368305843209, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.991413083090379, + "pass@2": 0.9955575945226233, + "pass@4": 0.9977777331405003, + "pass@8": 0.9991937098766038, + "pass@16": 0.9998749765033144, + "pass@32": 0.9999973906863244, + "pass@64": 0.9999999999217336, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9939692982456141, + "pass@2": 0.9942855107749691, + "pass@4": 0.994424011143344, + "pass@8": 0.9946902196435972, + "pass@16": 0.9952053690657088, + "pass@32": 0.9961665975963532, + "pass@64": 0.9978127734033246, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9451976102941176, + "pass@2": 0.9563129631773971, + "pass@4": 0.9654465556695121, + "pass@8": 0.9718827265720327, + "pass@16": 0.9754730011095328, + "pass@32": 0.9777955943914058, + "pass@64": 0.9800680829193935, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9673205835962145, + "pass@2": 0.9785331385528704, + "pass@4": 0.9846225014459944, + "pass@8": 0.9883663877814471, + "pass@16": 0.9908695441522511, + "pass@32": 0.9926653989314137, + "pass@64": 0.9935901740419045, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9580943433544303, + "pass@2": 0.9699429383035981, + "pass@4": 0.9774484110174518, + "pass@8": 0.9825297234375893, + "pass@16": 0.9862960040047856, + "pass@32": 0.9894226112977601, + "pass@64": 0.9918078764397377, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9388138138138138, + "pass@2": 0.9599656250738929, + "pass@4": 0.9730372833463381, + "pass@8": 0.982127536354344, + "pass@16": 0.9887220242201132, + "pass@32": 0.9932794012962612, + "pass@64": 0.9965889690345809, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9318227491961415, + "pass@2": 0.9592401796338966, + "pass@4": 0.9720742425283658, + "pass@8": 0.9775956122558883, + "pass@16": 0.9807653370853542, + "pass@32": 0.9840228942029388, + "pass@64": 0.9865052259333055, + "pass@128": 0.9871382636655949 + }, + "11": { + "pass@1": 0.8969246031746032, + "pass@2": 0.9353073053368325, + "pass@4": 0.9577159161652413, + "pass@8": 0.9715580156452523, + "pass@16": 0.9792506883459516, + "pass@32": 0.9836697037806984, + "pass@64": 0.9870159888279616, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8460196865443425, + "pass@2": 0.8954093073514888, + "pass@4": 0.921770653094968, + "pass@8": 0.9377565497507031, + "pass@16": 0.9494860703772144, + "pass@32": 0.9585818773272825, + "pass@64": 0.9648959273499919, + "pass@128": 0.9694189602446484 + }, + "13": { + "pass@1": 0.6845823688271605, + "pass@2": 0.7711701510401477, + "pass@4": 0.831109089373087, + "pass@8": 0.8697096758528202, + "pass@16": 0.8955431218247937, + "pass@32": 0.9153676932906923, + "pass@64": 0.9321230211088019, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.6283150964391692, + "pass@2": 0.721634121124325, + "pass@4": 0.7920033450937324, + "pass@8": 0.844320515659269, + "pass@16": 0.8806439842164706, + "pass@32": 0.9048698741234554, + "pass@64": 0.9205488513442519, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.5363859953703703, + "pass@2": 0.6341859719791972, + "pass@4": 0.713977784605628, + "pass@8": 0.7750546489304276, + "pass@16": 0.8188227494235403, + "pass@32": 0.8501646485533466, + "pass@64": 0.8744316116951758, + "pass@128": 0.8919753086419753 + }, + "16": { + "pass@1": 0.43372050147492625, + "pass@2": 0.5424948609852972, + "pass@4": 0.6338591301640388, + "pass@8": 0.7059378256142278, + "pass@16": 0.760447370706132, + "pass@32": 0.8038493559978188, + "pass@64": 0.8392871713523845, + "pass@128": 0.8702064896755162 + }, + "17": { + "pass@1": 0.331042921686747, + "pass@2": 0.42775160682098473, + "pass@4": 0.5139899623804555, + "pass@8": 0.5877757234896518, + "pass@16": 0.6526213506834679, + "pass@32": 0.7118324364171099, + "pass@64": 0.7624818371103406, + "pass@128": 0.8072289156626506 + }, + "18": { + "pass@1": 0.32104334677419355, + "pass@2": 0.41724465011430023, + "pass@4": 0.5019040053461057, + "pass@8": 0.5718798748514401, + "pass@16": 0.6303663786814787, + "pass@32": 0.6805273166490414, + "pass@64": 0.7261435356896053, + "pass@128": 0.7709677419354839 + }, + "19": { + "pass@1": 0.20907288519637462, + "pass@2": 0.2946860801198942, + "pass@4": 0.3835897539809034, + "pass@8": 0.4710165178235919, + "pass@16": 0.5532048615020811, + "pass@32": 0.623725705923306, + "pass@64": 0.6811089924313154, + "pass@128": 0.7250755287009063 + }, + "20": { + "pass@1": 0.19512481231231232, + "pass@2": 0.2750282270459435, + "pass@4": 0.35674344282415144, + "pass@8": 0.435467028303459, + "pass@16": 0.510642810713764, + "pass@32": 0.5809565839476649, + "pass@64": 0.6438056145361447, + "pass@128": 0.6966966966966966 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c9df006daaa8179b417ec75221e93cf3c9c44d3b --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3fa207a7e3b7bd86eaea8958003a4efd4ad64f3f3d59a4ff1330432f98d00d8 +size 2676542954 diff --git a/results/composition/SDAR-100M-block16-epoch1-op11-14/summary.csv b/results/composition/SDAR-100M-block16-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8b808c3a9f6919fc466f7bc8af2d08b492ec3eac --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op11-14,0.9566944444444444,0.510975,0.7221052631578947,0.0,0.0,0.0,201.07631163194444,362.007928125,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch1-op11-14/summary.json b/results/composition/SDAR-100M-block16-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c0a3dae6cbf5df95f6adaf51d87500fbf5898a28 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op11-14", + "id_acc": 0.9566944444444444, + "ood_acc": 0.510975, + "total_acc": 0.7221052631578947, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 201.07631163194444, + "ood_avg_resp_len": 362.007928125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 584256, + "answer_accuracy": 0.7215460006323111, + "avg_response_len": 272.650620949257, + "resp_tokens_sum": 220772842, + "pass_at_k": { + "pass@1": 0.7215460006323111, + "pass@2": 0.7726945850271698, + "pass@4": 0.8128004638992861, + "pass@8": 0.844156706546122, + "pass@16": 0.8689312388996039, + "pass@32": 0.8893245966044871, + "pass@64": 0.9068659649411043, + "pass@128": 0.9214353461903256 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9076327433628318, + "pass@2": 0.9269853436462037, + "pass@4": 0.9398492909625238, + "pass@8": 0.9506059829868715, + "pass@16": 0.9596484055654059, + "pass@32": 0.9658798721555774, + "pass@64": 0.9688322453458257, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9780957943925234, + "pass@2": 0.9850905757107465, + "pass@4": 0.9894686271692673, + "pass@8": 0.9920504616490534, + "pass@16": 0.9933079829828495, + "pass@32": 0.9937265331983628, + "pass@64": 0.9937693846777408, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.991829128440367, + "pass@2": 0.9972839010089335, + "pass@4": 0.9991199451559382, + "pass@8": 0.9998000718183061, + "pass@16": 0.9999896701598865, + "pass@32": 0.999999988244179, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9865234375, + "pass@2": 0.9907341904527559, + "pass@4": 0.9941022271630107, + "pass@8": 0.9962775829156062, + "pass@16": 0.997180906859181, + "pass@32": 0.9976553064004617, + "pass@64": 0.9984374999950079, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9469990079365079, + "pass@2": 0.9564476315460567, + "pass@4": 0.9630626984126983, + "pass@8": 0.9694378790828586, + "pass@16": 0.9759160456546958, + "pass@32": 0.9818998343552849, + "pass@64": 0.9875177250605657, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.926529752994012, + "pass@2": 0.9463030140506391, + "pass@4": 0.9610347533279894, + "pass@8": 0.9741423767959982, + "pass@16": 0.9846537870946782, + "pass@32": 0.9919116519382953, + "pass@64": 0.996595146234816, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9607063609467456, + "pass@2": 0.9766032678330153, + "pass@4": 0.9867770337879364, + "pass@8": 0.9932539965998115, + "pass@16": 0.9966691243987713, + "pass@32": 0.997758364804311, + "pass@64": 0.9985207077144822, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9363231169871795, + "pass@2": 0.9618858677821527, + "pass@4": 0.9778153304755176, + "pass@8": 0.9872242972247056, + "pass@16": 0.9919063397806129, + "pass@32": 0.9944326739175762, + "pass@64": 0.9972112783705197, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9345290492957746, + "pass@2": 0.9633556753909281, + "pass@4": 0.977406277032272, + "pass@8": 0.9842799230966626, + "pass@16": 0.9890320300163855, + "pass@32": 0.9926185701753283, + "pass@64": 0.9956966535148944, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8755975566572238, + "pass@2": 0.9161817436595214, + "pass@4": 0.9432045066604631, + "pass@8": 0.9603687852296585, + "pass@16": 0.9706278871438179, + "pass@32": 0.9775053297819941, + "pass@64": 0.981852619948499, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8450300141242938, + "pass@2": 0.8938623965701327, + "pass@4": 0.9229093023859299, + "pass@8": 0.9409758597723392, + "pass@16": 0.9516659432562773, + "pass@32": 0.9588886011978869, + "pass@64": 0.965495662578892, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7161295062695925, + "pass@2": 0.7977161651815466, + "pass@4": 0.8520592728612683, + "pass@8": 0.8873050572913995, + "pass@16": 0.9106471139096689, + "pass@32": 0.9292828710771102, + "pass@64": 0.9443766484455557, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.6268983644859814, + "pass@2": 0.7150230272033751, + "pass@4": 0.7842801770105835, + "pass@8": 0.837397655361588, + "pass@16": 0.8762786423304784, + "pass@32": 0.9046818450843452, + "pass@64": 0.9289289086935143, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.5480717124277457, + "pass@2": 0.6457133835510445, + "pass@4": 0.7187044204373292, + "pass@8": 0.7727291108200902, + "pass@16": 0.8139621475751042, + "pass@32": 0.8458336523771858, + "pass@64": 0.8698378477573592, + "pass@128": 0.8872832369942196 + }, + "16": { + "pass@1": 0.4441280241935484, + "pass@2": 0.5569766795783586, + "pass@4": 0.6460751194407147, + "pass@8": 0.7137075598181689, + "pass@16": 0.7681949863235685, + "pass@32": 0.8142011454810046, + "pass@64": 0.8517005897273575, + "pass@128": 0.8774193548387097 + }, + "17": { + "pass@1": 0.36806809971098264, + "pass@2": 0.4666509973373991, + "pass@4": 0.5472581679638306, + "pass@8": 0.6127021215261312, + "pass@16": 0.6691159679443502, + "pass@32": 0.7203100001054069, + "pass@64": 0.7654746436146028, + "pass@128": 0.8063583815028902 + }, + "18": { + "pass@1": 0.304375, + "pass@2": 0.3967902840269967, + "pass@4": 0.47884594809577347, + "pass@8": 0.5467893711314302, + "pass@16": 0.6037676808016317, + "pass@32": 0.6536113740422012, + "pass@64": 0.6999131573501903, + "pass@128": 0.74 + }, + "19": { + "pass@1": 0.21579391891891891, + "pass@2": 0.3006612665815421, + "pass@4": 0.38616035720760156, + "pass@8": 0.46550902965231605, + "pass@16": 0.5350209172548729, + "pass@32": 0.595347618281271, + "pass@64": 0.6482623365836448, + "pass@128": 0.6936936936936937 + }, + "20": { + "pass@1": 0.22670326576576577, + "pass@2": 0.3139590131470051, + "pass@4": 0.39960391071611545, + "pass@8": 0.4769628415256585, + "pass@16": 0.5428805161447351, + "pass@32": 0.6007913078306004, + "pass@64": 0.6557323133549934, + "pass@128": 0.7027027027027027 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 594054, + "answer_accuracy": 0.7234679462197974, + "avg_response_len": 282.07458349571317, + "resp_tokens_sum": 231617082, + "pass_at_k": { + "pass@1": 0.7234679462197974, + "pass@2": 0.7721883227671293, + "pass@4": 0.8113025592962161, + "pass@8": 0.8425025384347558, + "pass@16": 0.8674617623092497, + "pass@32": 0.8882591568285177, + "pass@64": 0.9060958084492297, + "pass@128": 0.9214341387373344 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8978144171779141, + "pass@2": 0.9213515500458915, + "pass@4": 0.9390322137646904, + "pass@8": 0.9526189121876671, + "pass@16": 0.9621797529616819, + "pass@32": 0.9684335022171842, + "pass@64": 0.9742357566082435, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.991950037993921, + "pass@2": 0.9956385521623626, + "pass@4": 0.996789173769692, + "pass@8": 0.9969556274713829, + "pass@16": 0.996960482749072, + "pass@32": 0.9969604863221885, + "pass@64": 0.9969604863221885, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9910748106060606, + "pass@2": 0.9938342579336673, + "pass@4": 0.9954610878185681, + "pass@8": 0.9967861359234476, + "pass@16": 0.9983705508051556, + "pass@32": 0.999643213120596, + "pass@64": 0.99999359856941, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9862934541420119, + "pass@2": 0.9914085431906072, + "pass@4": 0.9944976862174774, + "pass@8": 0.9968238286910404, + "pass@16": 0.9986471171735518, + "pass@32": 0.9997422779717037, + "pass@64": 0.9999955420471618, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9700860507246377, + "pass@2": 0.9768297814675341, + "pass@4": 0.9813448272770252, + "pass@8": 0.9845774240321683, + "pass@16": 0.9867515567319303, + "pass@32": 0.9884426560783856, + "pass@64": 0.9902316406247298, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9678545845272206, + "pass@2": 0.9800999199061439, + "pass@4": 0.9855557597742974, + "pass@8": 0.988306274970953, + "pass@16": 0.9904143515720015, + "pass@32": 0.9923533345945439, + "pass@64": 0.9938368819680918, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.965904985549133, + "pass@2": 0.9778386850848846, + "pass@4": 0.9852525098018816, + "pass@8": 0.9905732375817846, + "pass@16": 0.9943721417100441, + "pass@32": 0.997168595303731, + "pass@64": 0.9991059501069279, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9451804577464789, + "pass@2": 0.9669578989686152, + "pass@4": 0.9803207029579047, + "pass@8": 0.9881750952049374, + "pass@16": 0.9925613357660423, + "pass@32": 0.9958566064759846, + "pass@64": 0.9987507974877009, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9378508607784432, + "pass@2": 0.960500259323872, + "pass@4": 0.9725681423991157, + "pass@8": 0.9806556822439841, + "pass@16": 0.9856425911302683, + "pass@32": 0.988864619985313, + "pass@64": 0.9917644684832534, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9046733810240963, + "pass@2": 0.9394221818375864, + "pass@4": 0.9586932646220422, + "pass@8": 0.9705318114541525, + "pass@16": 0.9775325668360785, + "pass@32": 0.9814679960471665, + "pass@64": 0.9848638566163492, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8320190047021944, + "pass@2": 0.8726735862562636, + "pass@4": 0.8965225433223666, + "pass@8": 0.9119176475126256, + "pass@16": 0.9216786817337147, + "pass@32": 0.9290268632604375, + "pass@64": 0.9362206325610609, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.6824010854341737, + "pass@2": 0.7620663915172362, + "pass@4": 0.816710596574587, + "pass@8": 0.8568656338449239, + "pass@16": 0.8863689471035519, + "pass@32": 0.9093439500882927, + "pass@64": 0.929124181128099, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6183982090643275, + "pass@2": 0.7113501231753926, + "pass@4": 0.7827317758525795, + "pass@8": 0.8349723554689547, + "pass@16": 0.8735588597859756, + "pass@32": 0.9025529986048424, + "pass@64": 0.9217735735634356, + "pass@128": 0.9327485380116959 + }, + "15": { + "pass@1": 0.5301846590909091, + "pass@2": 0.6305125566690525, + "pass@4": 0.7065275675767798, + "pass@8": 0.7606926843299037, + "pass@16": 0.7990498123628644, + "pass@32": 0.8293580860368167, + "pass@64": 0.8559740595796936, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.4334045584045584, + "pass@2": 0.5336198459968144, + "pass@4": 0.6156393698650913, + "pass@8": 0.6793719515351224, + "pass@16": 0.7282840734236551, + "pass@32": 0.7671109052665265, + "pass@64": 0.800987680746752, + "pass@128": 0.8319088319088319 + }, + "17": { + "pass@1": 0.3566818711180124, + "pass@2": 0.46245017606494854, + "pass@4": 0.5551400752700942, + "pass@8": 0.6291890200102845, + "pass@16": 0.6878346867297733, + "pass@32": 0.7384604652449943, + "pass@64": 0.7881347414072649, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.32173713235294116, + "pass@2": 0.42347802802223233, + "pass@4": 0.5183494841637442, + "pass@8": 0.5990558759706359, + "pass@16": 0.6651894072773802, + "pass@32": 0.7191764944894445, + "pass@64": 0.7606179403198643, + "pass@128": 0.7911764705882353 + }, + "19": { + "pass@1": 0.20986793154761904, + "pass@2": 0.2867501669713159, + "pass@4": 0.3652424758289142, + "pass@8": 0.4410855714382008, + "pass@16": 0.5130674911401856, + "pass@32": 0.5803165592539755, + "pass@64": 0.6393250538858533, + "pass@128": 0.6934523809523809 + }, + "20": { + "pass@1": 0.18988585329341318, + "pass@2": 0.2721880304351926, + "pass@4": 0.35637058588110615, + "pass@8": 0.4371482552763727, + "pass@16": 0.512849984475913, + "pass@32": 0.5830933497627412, + "pass@64": 0.645622648235715, + "pass@128": 0.6976047904191617 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 577850, + "answer_accuracy": 0.7212738656334878, + "avg_response_len": 302.8390804741173, + "resp_tokens_sum": 242620135, + "pass_at_k": { + "pass@1": 0.7212738656334878, + "pass@2": 0.7713678130264008, + "pass@4": 0.8116175308081232, + "pass@8": 0.8437661082162949, + "pass@16": 0.8696224555542752, + "pass@32": 0.8909510236880488, + "pass@64": 0.9086669802588497, + "pass@128": 0.9234702029078128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8983908582089553, + "pass@2": 0.9294809172640736, + "pass@4": 0.950525032318721, + "pass@8": 0.9653934920081807, + "pass@16": 0.9748415411280041, + "pass@32": 0.9810193030176554, + "pass@64": 0.9861676540954208, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9804464285714286, + "pass@2": 0.9854861501687289, + "pass@4": 0.9888010849536667, + "pass@8": 0.9906410014232532, + "pass@16": 0.9919181611918471, + "pass@32": 0.99303540900536, + "pass@64": 0.9939368305843209, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.991413083090379, + "pass@2": 0.9955575945226233, + "pass@4": 0.9977777331405003, + "pass@8": 0.9991937098766038, + "pass@16": 0.9998749765033144, + "pass@32": 0.9999973906863244, + "pass@64": 0.9999999999217336, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9939692982456141, + "pass@2": 0.9942855107749691, + "pass@4": 0.994424011143344, + "pass@8": 0.9946902196435972, + "pass@16": 0.9952053690657088, + "pass@32": 0.9961665975963532, + "pass@64": 0.9978127734033246, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9451976102941176, + "pass@2": 0.9563129631773971, + "pass@4": 0.9654465556695121, + "pass@8": 0.9718827265720327, + "pass@16": 0.9754730011095328, + "pass@32": 0.9777955943914058, + "pass@64": 0.9800680829193935, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9673205835962145, + "pass@2": 0.9785331385528704, + "pass@4": 0.9846225014459944, + "pass@8": 0.9883663877814471, + "pass@16": 0.9908695441522511, + "pass@32": 0.9926653989314137, + "pass@64": 0.9935901740419045, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9580943433544303, + "pass@2": 0.9699429383035981, + "pass@4": 0.9774484110174518, + "pass@8": 0.9825297234375893, + "pass@16": 0.9862960040047856, + "pass@32": 0.9894226112977601, + "pass@64": 0.9918078764397377, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9388138138138138, + "pass@2": 0.9599656250738929, + "pass@4": 0.9730372833463381, + "pass@8": 0.982127536354344, + "pass@16": 0.9887220242201132, + "pass@32": 0.9932794012962612, + "pass@64": 0.9965889690345809, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9318227491961415, + "pass@2": 0.9592401796338966, + "pass@4": 0.9720742425283658, + "pass@8": 0.9775956122558883, + "pass@16": 0.9807653370853542, + "pass@32": 0.9840228942029388, + "pass@64": 0.9865052259333055, + "pass@128": 0.9871382636655949 + }, + "11": { + "pass@1": 0.8969246031746032, + "pass@2": 0.9353073053368325, + "pass@4": 0.9577159161652413, + "pass@8": 0.9715580156452523, + "pass@16": 0.9792506883459516, + "pass@32": 0.9836697037806984, + "pass@64": 0.9870159888279616, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8460196865443425, + "pass@2": 0.8954093073514888, + "pass@4": 0.921770653094968, + "pass@8": 0.9377565497507031, + "pass@16": 0.9494860703772144, + "pass@32": 0.9585818773272825, + "pass@64": 0.9648959273499919, + "pass@128": 0.9694189602446484 + }, + "13": { + "pass@1": 0.6845823688271605, + "pass@2": 0.7711701510401477, + "pass@4": 0.831109089373087, + "pass@8": 0.8697096758528202, + "pass@16": 0.8955431218247937, + "pass@32": 0.9153676932906923, + "pass@64": 0.9321230211088019, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.6283150964391692, + "pass@2": 0.721634121124325, + "pass@4": 0.7920033450937324, + "pass@8": 0.844320515659269, + "pass@16": 0.8806439842164706, + "pass@32": 0.9048698741234554, + "pass@64": 0.9205488513442519, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.5363859953703703, + "pass@2": 0.6341859719791972, + "pass@4": 0.713977784605628, + "pass@8": 0.7750546489304276, + "pass@16": 0.8188227494235403, + "pass@32": 0.8501646485533466, + "pass@64": 0.8744316116951758, + "pass@128": 0.8919753086419753 + }, + "16": { + "pass@1": 0.43372050147492625, + "pass@2": 0.5424948609852972, + "pass@4": 0.6338591301640388, + "pass@8": 0.7059378256142278, + "pass@16": 0.760447370706132, + "pass@32": 0.8038493559978188, + "pass@64": 0.8392871713523845, + "pass@128": 0.8702064896755162 + }, + "17": { + "pass@1": 0.331042921686747, + "pass@2": 0.42775160682098473, + "pass@4": 0.5139899623804555, + "pass@8": 0.5877757234896518, + "pass@16": 0.6526213506834679, + "pass@32": 0.7118324364171099, + "pass@64": 0.7624818371103406, + "pass@128": 0.8072289156626506 + }, + "18": { + "pass@1": 0.32104334677419355, + "pass@2": 0.41724465011430023, + "pass@4": 0.5019040053461057, + "pass@8": 0.5718798748514401, + "pass@16": 0.6303663786814787, + "pass@32": 0.6805273166490414, + "pass@64": 0.7261435356896053, + "pass@128": 0.7709677419354839 + }, + "19": { + "pass@1": 0.20907288519637462, + "pass@2": 0.2946860801198942, + "pass@4": 0.3835897539809034, + "pass@8": 0.4710165178235919, + "pass@16": 0.5532048615020811, + "pass@32": 0.623725705923306, + "pass@64": 0.6811089924313154, + "pass@128": 0.7250755287009063 + }, + "20": { + "pass@1": 0.19512481231231232, + "pass@2": 0.2750282270459435, + "pass@4": 0.35674344282415144, + "pass@8": 0.435467028303459, + "pass@16": 0.510642810713764, + "pass@32": 0.5809565839476649, + "pass@64": 0.6438056145361447, + "pass@128": 0.6966966966966966 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5b586c70bf0ac2b785f81323e17cde68a909c24c --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2fa6ee4b9228a31951910b169eae81bfbda21ef687b8869e2da069a264d0bef +size 1796825349 diff --git a/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json b/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a6cc0f79814beec0ac3e87b9927eae16dd68ab93 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9134383680555556, + "avg_response_len": 202.60357899305555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9134383680555556, + "pass@2": 0.9481420466972658, + "pass@4": 0.966789149585467, + "pass@8": 0.9779535209475929, + "pass@16": 0.9851560966687626, + "pass@32": 0.989967281322493, + "pass@64": 0.9933028645399439, + "pass@128": 0.9957777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8259609375, + "pass@2": 0.893026451771654, + "pass@4": 0.9322231326396698, + "pass@8": 0.9565009551603775, + "pass@16": 0.971507385154414, + "pass@32": 0.9808318511743037, + "pass@64": 0.9876695059302211, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.935625, + "pass@2": 0.9512100147637794, + "pass@4": 0.9636213038995125, + "pass@8": 0.9727229765191304, + "pass@16": 0.978889970330367, + "pass@32": 0.982836458697959, + "pass@64": 0.9851793710344321, + "pass@128": 0.987 + }, + "3": { + "pass@1": 0.977046875, + "pass@2": 0.9854843750000004, + "pass@4": 0.9901338926696664, + "pass@8": 0.9932631652553967, + "pass@16": 0.9954292364313113, + "pass@32": 0.9967808357169232, + "pass@64": 0.9977499040291772, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.97296875, + "pass@2": 0.9835835383858268, + "pass@4": 0.9883603146794149, + "pass@8": 0.9911434994665775, + "pass@16": 0.9930824160789711, + "pass@32": 0.9947548746152566, + "pass@64": 0.9964802260461338, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9220625, + "pass@2": 0.9477059547244095, + "pass@4": 0.963529338582677, + "pass@8": 0.9743781813113109, + "pass@16": 0.9822554465747628, + "pass@32": 0.9885125209347927, + "pass@64": 0.9929442929712965, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.9371796875, + "pass@2": 0.9675015994094499, + "pass@4": 0.9815036453880766, + "pass@8": 0.9885290906061706, + "pass@16": 0.9921437299044742, + "pass@32": 0.99416612649408, + "pass@64": 0.995942199982249, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.908703125, + "pass@2": 0.9471446850393703, + "pass@4": 0.967179319928759, + "pass@8": 0.9779200100902848, + "pass@16": 0.9847850762647822, + "pass@32": 0.9895726876030048, + "pass@64": 0.9928923860328146, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.8838984375, + "pass@2": 0.9365091043307101, + "pass@4": 0.9626962175665545, + "pass@8": 0.9774632890377969, + "pass@16": 0.9864115667605743, + "pass@32": 0.9920013654389653, + "pass@64": 0.9956127630881643, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.8575, + "pass@2": 0.9211126968503948, + "pass@4": 0.9518551809148847, + "pass@8": 0.9696605210813218, + "pass@16": 0.9819000425192099, + "pass@32": 0.9902488112271479, + "pass@64": 0.9952551317449962, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.8259609375, + "2": 0.935625, + "3": 0.977046875, + "4": 0.97296875, + "5": 0.9220625, + "6": 0.9371796875, + "7": 0.908703125, + "8": 0.8838984375, + "9": 0.8575 + }, + "per_op_avg_response_len": { + "10": 281.24259375, + "2": 122.4647578125, + "3": 146.4731015625, + "4": 160.6047265625, + "5": 176.30828125, + "6": 201.5447421875, + "7": 226.3348359375, + "8": 246.195390625, + "9": 262.26378125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.24259375, + "2": 122.4647578125, + "3": 146.4731015625, + "4": 160.6047265625, + "5": 176.30828125, + "6": 201.5447421875, + "7": 226.3348359375, + "8": 246.195390625, + "9": 262.26378125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 344132, + "answer_accuracy": 0.9079808341776426, + "avg_response_len": 192.52982786643025, + "resp_tokens_sum": 72970345, + "pass_at_k": { + "pass@1": 0.9079808341776426, + "pass@2": 0.945897512478498, + "pass@4": 0.966108559188462, + "pass@8": 0.9778946033419159, + "pass@16": 0.9851827312761958, + "pass@32": 0.9899476901168374, + "pass@64": 0.9931722876009214, + "pass@128": 0.9956095913542722 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8337020648967551, + "pass@2": 0.8992248652823267, + "pass@4": 0.9340549527547986, + "pass@8": 0.9549133337554659, + "pass@16": 0.9676459667453204, + "pass@32": 0.9754629221119615, + "pass@64": 0.9816283664853948, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9328514408099688, + "pass@2": 0.9459657934113376, + "pass@4": 0.9572665329801058, + "pass@8": 0.9661779608434554, + "pass@16": 0.9726776857540342, + "pass@32": 0.9771842431316051, + "pass@64": 0.9796170198954688, + "pass@128": 0.9813084112149533 + }, + "3": { + "pass@1": 0.9756068425076453, + "pass@2": 0.9874613222085766, + "pass@4": 0.9940523552675177, + "pass@8": 0.9974856871737064, + "pass@16": 0.9991656932084267, + "pass@32": 0.9998836449467822, + "pass@64": 0.9999990650219919, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.975341796875, + "pass@2": 0.9855330339566928, + "pass@4": 0.9891930282152229, + "pass@8": 0.9909968147034551, + "pass@16": 0.9920335522380169, + "pass@32": 0.9932232658217899, + "pass@64": 0.9949311023622034, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9057291666666667, + "pass@2": 0.9370816929133859, + "pass@4": 0.9557188610352269, + "pass@8": 0.9680085338015769, + "pass@16": 0.9772068251757668, + "pass@32": 0.9850956818037647, + "pass@64": 0.9907377001622719, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9124719311377245, + "pass@2": 0.9472194881889765, + "pass@4": 0.967283268951411, + "pass@8": 0.9804620871830513, + "pass@16": 0.9883802999024801, + "pass@32": 0.9926513268166495, + "pass@64": 0.9960828629239575, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8930981878698225, + "pass@2": 0.9425206314354944, + "pass@4": 0.9694562882857096, + "pass@8": 0.9828592027889445, + "pass@16": 0.9902744534062847, + "pass@32": 0.9950889757930396, + "pass@64": 0.9980455562464515, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8852413862179487, + "pass@2": 0.9402193115283668, + "pass@4": 0.9669937225394901, + "pass@8": 0.9816570277298262, + "pass@16": 0.990420934947623, + "pass@32": 0.996331498679127, + "pass@64": 0.9995570396284617, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8653389084507043, + "pass@2": 0.9311758345347674, + "pass@4": 0.9623128128350147, + "pass@8": 0.9789012564778983, + "pass@16": 0.9887982028366319, + "pass@32": 0.994471708460012, + "pass@64": 0.9977497769018057, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 358896, + "answer_accuracy": 0.9187008519003932, + "avg_response_len": 200.21991214777196, + "resp_tokens_sum": 78217110, + "pass_at_k": { + "pass@1": 0.9187008519003932, + "pass@2": 0.9517420243986168, + "pass@4": 0.9687024149831882, + "pass@8": 0.9789000894335694, + "pass@16": 0.9858622215313799, + "pass@32": 0.9907073713245071, + "pass@64": 0.9940006009808476, + "pass@128": 0.9960681520314548 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8397478911042945, + "pass@2": 0.9017401031351142, + "pass@4": 0.9367013483935669, + "pass@8": 0.9597981710002264, + "pass@16": 0.9760430566524732, + "pass@32": 0.9868893358013864, + "pass@64": 0.9939010380542732, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9402308130699089, + "pass@2": 0.9559446275997415, + "pass@4": 0.9671450020115263, + "pass@8": 0.9747791046336394, + "pass@16": 0.9798526018916596, + "pass@32": 0.982735766252235, + "pass@64": 0.9843437813422433, + "pass@128": 0.9848024316109423 + }, + "3": { + "pass@1": 0.98203125, + "pass@2": 0.9875126759723218, + "pass@4": 0.9907982465260023, + "pass@8": 0.9931905013110562, + "pass@16": 0.9947669019933483, + "pass@32": 0.996016072122554, + "pass@64": 0.9977332350126279, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9693047337278107, + "pass@2": 0.9827249947584216, + "pass@4": 0.9890199579342521, + "pass@8": 0.9924316167169746, + "pass@16": 0.9952105268225401, + "pass@32": 0.9977446704990522, + "pass@64": 0.9995571159589419, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.948052536231884, + "pass@2": 0.966407052379322, + "pass@4": 0.9765803837020368, + "pass@8": 0.9844381946054411, + "pass@16": 0.9908400253696799, + "pass@32": 0.9954288349662425, + "pass@64": 0.9982494068365969, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9469914040114613, + "pass@2": 0.9771242498251473, + "pass@4": 0.9882326420336424, + "pass@8": 0.991931583238474, + "pass@16": 0.9933659205077896, + "pass@32": 0.9940735913681014, + "pass@64": 0.9942652013230373, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9118722904624278, + "pass@2": 0.9468681728187156, + "pass@4": 0.9655806095047366, + "pass@8": 0.9770071107892204, + "pass@16": 0.9847477489910627, + "pass@32": 0.9900681636945455, + "pass@64": 0.9936873526856657, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.8875, + "pass@2": 0.9402195159143837, + "pass@4": 0.9639932780409486, + "pass@8": 0.9756221536661233, + "pass@16": 0.9821909911786181, + "pass@32": 0.987287230265571, + "pass@64": 0.992438504371287, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.8411302395209581, + "pass@2": 0.905336104366071, + "pass@4": 0.938773440464403, + "pass@8": 0.9598987045631255, + "pass@16": 0.9751816882329573, + "pass@32": 0.9858225193162448, + "pass@64": 0.9916186385525599, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 349253, + "answer_accuracy": 0.9134713968865082, + "avg_response_len": 215.02518203883494, + "resp_tokens_sum": 82211868, + "pass_at_k": { + "pass@1": 0.9134713968865082, + "pass@2": 0.9466887269374664, + "pass@4": 0.9655089159636779, + "pass@8": 0.9770447589828205, + "pass@16": 0.9844082030787714, + "pass@32": 0.9892305069247003, + "pass@64": 0.9927193850283234, + "pass@128": 0.995647807164379 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8047108208955224, + "pass@2": 0.8782744740862615, + "pass@4": 0.9260115345656411, + "pass@8": 0.9548988993170184, + "pass@16": 0.9710010924150604, + "pass@32": 0.980370140618196, + "pass@64": 0.9877186605551607, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9338392857142858, + "pass@2": 0.9515691788526434, + "pass@4": 0.9661372604317318, + "pass@8": 0.9767929304683264, + "pass@16": 0.9836826490884742, + "pass@32": 0.9881149987306532, + "pass@64": 0.9910662959611092, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9736242711370262, + "pass@2": 0.9816482203117467, + "pass@4": 0.9857590412350061, + "pass@8": 0.9893075222068398, + "pass@16": 0.9925043063975238, + "pass@32": 0.9945585423291038, + "pass@64": 0.9956216974134651, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9743695175438597, + "pass@2": 0.9826079511442651, + "pass@4": 0.9869292393823577, + "pass@8": 0.9900076968161828, + "pass@16": 0.9919605885870972, + "pass@32": 0.9932331313555678, + "pass@64": 0.9948887956026504, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9108226102941176, + "pass@2": 0.9385729070171378, + "pass@4": 0.9575225440569924, + "pass@8": 0.9700715177204019, + "pass@16": 0.9782220232113423, + "pass@32": 0.9846600973625091, + "pass@64": 0.9896055060633964, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9524102917981072, + "pass@2": 0.9782773727613706, + "pass@4": 0.9890783643172356, + "pass@8": 0.9932827474347146, + "pass@16": 0.994763417917439, + "pass@32": 0.9958640376966922, + "pass@64": 0.9976402791921362, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9219244462025317, + "pass@2": 0.9523934298066383, + "pass@4": 0.9664943151884493, + "pass@8": 0.973636516501745, + "pass@16": 0.9789543982992107, + "pass@32": 0.983129839071662, + "pass@64": 0.9865100126337769, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.8788006756756757, + "pass@2": 0.9290773302830391, + "pass@4": 0.9572869742633514, + "pass@8": 0.9754967922961485, + "pass@16": 0.9871544540195394, + "pass@32": 0.992969883203604, + "pass@64": 0.9953011942110427, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.8661324356913184, + "pass@2": 0.9265691856343519, + "pass@4": 0.9539672130011069, + "pass@8": 0.9695961662623275, + "pass@16": 0.9812411467601234, + "pass@32": 0.9901821005537559, + "pass@64": 0.9963129764254025, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.381703125, + "avg_response_len": 351.9533, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.381703125, + "pass@2": 0.4851356299212578, + "pass@4": 0.57571848824521, + "pass@8": 0.6504443915725758, + "pass@16": 0.7106175845483492, + "pass@32": 0.7599688087801404, + "pass@64": 0.8018412728035076, + "pass@128": 0.8372 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7578203125, + "pass@2": 0.847205954724411, + "pass@4": 0.9011952808398942, + "pass@8": 0.93377580565178, + "pass@16": 0.955070876530586, + "pass@32": 0.969590082180953, + "pass@64": 0.9787324423322356, + "pass@128": 0.983 + }, + "12": { + "pass@1": 0.641609375, + "pass@2": 0.7585755413385825, + "pass@4": 0.8381425514623176, + "pass@8": 0.8892629624537244, + "pass@16": 0.9220148867551953, + "pass@32": 0.9435080487950828, + "pass@64": 0.9590211028900316, + "pass@128": 0.972 + }, + "13": { + "pass@1": 0.465140625, + "pass@2": 0.5931075295275604, + "pass@4": 0.6977997386576684, + "pass@8": 0.7762012373108262, + "pass@16": 0.8331259756536721, + "pass@32": 0.8759206686558857, + "pass@64": 0.9090539128333244, + "pass@128": 0.932 + }, + "14": { + "pass@1": 0.43965625, + "pass@2": 0.5558090551181101, + "pass@4": 0.657769242407198, + "pass@8": 0.7426246540339614, + "pass@16": 0.8089162040581582, + "pass@32": 0.8580974144957311, + "pass@64": 0.8936584566878355, + "pass@128": 0.919 + }, + "15": { + "pass@1": 0.3664609375, + "pass@2": 0.49034571850393777, + "pass@4": 0.598005425571804, + "pass@8": 0.6826972018891915, + "pass@16": 0.7454130715416146, + "pass@32": 0.7926700168179653, + "pass@64": 0.8309058005394662, + "pass@128": 0.862 + }, + "16": { + "pass@1": 0.281734375, + "pass@2": 0.3871161417322831, + "pass@4": 0.4866833546119234, + "pass@8": 0.5734005358977272, + "pass@16": 0.6484476727811473, + "pass@32": 0.7138604538727543, + "pass@64": 0.7698033716622331, + "pass@128": 0.817 + }, + "17": { + "pass@1": 0.2525, + "pass@2": 0.35274815452755964, + "pass@4": 0.4507468760779903, + "pass@8": 0.5374077389510419, + "pass@16": 0.6103234449582415, + "pass@32": 0.6721658021677973, + "pass@64": 0.7278754450284157, + "pass@128": 0.781 + }, + "18": { + "pass@1": 0.2393671875, + "pass@2": 0.337944881889764, + "pass@4": 0.435632553618297, + "pass@8": 0.5206967651312548, + "pass@16": 0.5906179695379268, + "pass@32": 0.6484050092386298, + "pass@64": 0.6969525325161366, + "pass@128": 0.736 + }, + "19": { + "pass@1": 0.188734375, + "pass@2": 0.2653257874015747, + "pass@4": 0.3448389823772027, + "pass@8": 0.42249623713425216, + "pass@16": 0.4957875266004969, + "pass@32": 0.5655016837307463, + "pass@64": 0.6324997305839606, + "pass@128": 0.695 + }, + "20": { + "pass@1": 0.1840078125, + "pass@2": 0.2631775344488189, + "pass@4": 0.34637087682789613, + "pass@8": 0.4258807772719833, + "pass@16": 0.49645821706625787, + "pass@32": 0.5599689078458404, + "pass@64": 0.6199099329613685, + "pass@128": 0.675 + } + }, + "per_op_accuracy": { + "11": 0.7578203125, + "12": 0.641609375, + "13": 0.465140625, + "14": 0.43965625, + "15": 0.3664609375, + "16": 0.281734375, + "17": 0.2525, + "18": 0.2393671875, + "19": 0.188734375, + "20": 0.1840078125 + }, + "per_op_avg_response_len": { + "11": 298.0571640625, + "12": 304.7676015625, + "13": 319.7301328125, + "14": 341.958609375, + "15": 356.6907265625, + "16": 359.9821875, + "17": 375.317109375, + "18": 383.491953125, + "19": 383.327109375, + "20": 396.21040625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.0571640625, + "12": 304.7676015625, + "13": 319.7301328125, + "14": 341.958609375, + "15": 356.6907265625, + "16": 359.9821875, + "17": 375.317109375, + "18": 383.491953125, + "19": 383.327109375, + "20": 396.21040625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 166367, + "answer_accuracy": 0.386253250371471, + "avg_response_len": 331.85956770059437, + "resp_tokens_sum": 142938553, + "pass_at_k": { + "pass@1": 0.386253250371471, + "pass@2": 0.4909063088649968, + "pass@4": 0.5805378291773731, + "pass@8": 0.6526600886267144, + "pass@16": 0.7094506722937122, + "pass@32": 0.7551306301579921, + "pass@64": 0.7938336063060175, + "pass@128": 0.8276374442793462 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7575690509915014, + "pass@2": 0.8498137449532688, + "pass@4": 0.9047521838629934, + "pass@8": 0.9377886732347469, + "pass@16": 0.960041149186458, + "pass@32": 0.9746239590826914, + "pass@64": 0.9825700799894423, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6333200918079096, + "pass@2": 0.7547513095555853, + "pass@4": 0.8386714274592795, + "pass@8": 0.8928341163275466, + "pass@16": 0.9259803364931931, + "pass@32": 0.9447383494093016, + "pass@64": 0.9572942446713538, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.4841300940438871, + "pass@2": 0.6205060721249966, + "pass@4": 0.7295860708438091, + "pass@8": 0.8059492526033689, + "pass@16": 0.8568992077900991, + "pass@32": 0.8937514227248922, + "pass@64": 0.9225152134466567, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.4497663551401869, + "pass@2": 0.571845027841146, + "pass@4": 0.670868385610677, + "pass@8": 0.7470698554728201, + "pass@16": 0.8054958898697235, + "pass@32": 0.8505787819273606, + "pass@64": 0.884048299310893, + "pass@128": 0.9065420560747663 + }, + "15": { + "pass@1": 0.36836163294797686, + "pass@2": 0.4893602646670612, + "pass@4": 0.5954076710425649, + "pass@8": 0.6802021581752167, + "pass@16": 0.7428461601629132, + "pass@32": 0.7885083115987753, + "pass@64": 0.8230281644538102, + "pass@128": 0.8497109826589595 + }, + "16": { + "pass@1": 0.28364415322580644, + "pass@2": 0.39256453200406405, + "pass@4": 0.4944607647834345, + "pass@8": 0.5805687140274347, + "pass@16": 0.6532026734319762, + "pass@32": 0.714863241409248, + "pass@64": 0.7669355763349044, + "pass@128": 0.8129032258064516 + }, + "17": { + "pass@1": 0.2501128973988439, + "pass@2": 0.34497284773109993, + "pass@4": 0.43463790797826557, + "pass@8": 0.5132459972107867, + "pass@16": 0.5819257752825826, + "pass@32": 0.643364780253834, + "pass@64": 0.7006827171900452, + "pass@128": 0.7543352601156069 + }, + "18": { + "pass@1": 0.23118303571428572, + "pass@2": 0.32596245781777267, + "pass@4": 0.41872846885210785, + "pass@8": 0.4994272862212975, + "pass@16": 0.5665476115510434, + "pass@32": 0.6217479887245179, + "pass@64": 0.6670019576727322, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18700731981981983, + "pass@2": 0.2625891147052565, + "pass@4": 0.34095073307278057, + "pass@8": 0.4179914747194478, + "pass@16": 0.48919832053560813, + "pass@32": 0.5569396106072487, + "pass@64": 0.6262402386445067, + "pass@128": 0.7027027027027027 + }, + "20": { + "pass@1": 0.19280217717717718, + "pass@2": 0.27425383060225567, + "pass@4": 0.3585156604861329, + "pass@8": 0.43630993945564095, + "pass@16": 0.5004092983315865, + "pass@32": 0.5534055957447018, + "pass@64": 0.6021505971017288, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 163345, + "answer_accuracy": 0.3794626263752602, + "avg_response_len": 347.8358074078204, + "resp_tokens_sum": 149730793, + "pass_at_k": { + "pass@1": 0.3794626263752602, + "pass@2": 0.48323833970185204, + "pass@4": 0.5751190968156217, + "pass@8": 0.6513041251105385, + "pass@16": 0.7129426720067712, + "pass@32": 0.7640267389110413, + "pass@64": 0.8065358289432528, + "pass@128": 0.8409158489443949 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7506353539156626, + "pass@2": 0.8404700247841763, + "pass@4": 0.8961459129544045, + "pass@8": 0.9278766949955667, + "pass@16": 0.9467320736664068, + "pass@32": 0.9597594455810305, + "pass@64": 0.9681259167521782, + "pass@128": 0.9728915662650602 + }, + "12": { + "pass@1": 0.6611726097178683, + "pass@2": 0.7715347542764048, + "pass@4": 0.8428349935881834, + "pass@8": 0.8867303652307457, + "pass@16": 0.9159569131757227, + "pass@32": 0.9389731417905509, + "pass@64": 0.9577601020248315, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.44421831232493, + "pass@2": 0.5660448234411872, + "pass@4": 0.66785195968151, + "pass@8": 0.7483170301361325, + "pass@16": 0.8105249949265751, + "pass@32": 0.8597446273776105, + "pass@64": 0.8981030316546639, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.4415433114035088, + "pass@2": 0.5532334260026704, + "pass@4": 0.652206162826138, + "pass@8": 0.737487007203177, + "pass@16": 0.8066073874498777, + "pass@32": 0.8591202423066083, + "pass@64": 0.8958383932620966, + "pass@128": 0.9210526315789473 + }, + "15": { + "pass@1": 0.34607007575757576, + "pass@2": 0.4742386960152705, + "pass@4": 0.5880804396609509, + "pass@8": 0.6771553828886003, + "pass@16": 0.7422639165081902, + "pass@32": 0.7913001834008943, + "pass@64": 0.8312857431811411, + "pass@128": 0.8636363636363636 + }, + "16": { + "pass@1": 0.2948940527065527, + "pass@2": 0.4010665533795455, + "pass@4": 0.49861933924926044, + "pass@8": 0.5817780568854317, + "pass@16": 0.6520585361736072, + "pass@32": 0.7131258389048448, + "pass@64": 0.7662778139871184, + "pass@128": 0.811965811965812 + }, + "17": { + "pass@1": 0.2685364906832298, + "pass@2": 0.37610842972074143, + "pass@4": 0.48323010506838776, + "pass@8": 0.577938445005628, + "pass@16": 0.653921104286253, + "pass@32": 0.7138194380494706, + "pass@64": 0.7643783394471984, + "pass@128": 0.8136645962732919 + }, + "18": { + "pass@1": 0.24377297794117647, + "pass@2": 0.3515556247105141, + "pass@4": 0.46032718851320026, + "pass@8": 0.5538321588382297, + "pass@16": 0.6284314018506372, + "pass@32": 0.6875919900487388, + "pass@64": 0.7341714824652724, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.1875, + "pass@2": 0.2628534958520808, + "pass@4": 0.34137497935525907, + "pass@8": 0.41886761741849504, + "pass@16": 0.4948485199164589, + "pass@32": 0.5684772948464838, + "pass@64": 0.6340076592011885, + "pass@128": 0.6845238095238095 + }, + "20": { + "pass@1": 0.16871725299401197, + "pass@2": 0.24689621740770412, + "pass@4": 0.3304425002638143, + "pass@8": 0.41089814433551575, + "pass@16": 0.483789637592383, + "pass@32": 0.5521726444199829, + "pass@64": 0.6178883001231452, + "pass@128": 0.6766467065868264 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 158868, + "answer_accuracy": 0.37932648227383864, + "avg_response_len": 376.8501633175428, + "resp_tokens_sum": 157830878, + "pass_at_k": { + "pass@1": 0.37932648227383864, + "pass@2": 0.4811509882226685, + "pass@4": 0.5713782288140037, + "pass@8": 0.6472820735788902, + "pass@16": 0.7094279117531237, + "pass@32": 0.7607737146582738, + "pass@64": 0.8052514211733067, + "pass@128": 0.8432151589242054 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7656746031746032, + "pass@2": 0.8513830302462192, + "pass@4": 0.9025311645568108, + "pass@8": 0.9354963214647195, + "pass@16": 0.9582898489222895, + "pass@32": 0.974310122831304, + "pass@64": 0.9856107928071083, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.6314984709480123, + "pass@2": 0.7500733674059092, + "pass@4": 0.8329923644865482, + "pass@8": 0.887867580321604, + "pass@16": 0.923631780836543, + "pass@32": 0.946600124993897, + "pass@64": 0.9621206964233943, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.4694974922839506, + "pass@2": 0.5959508967629046, + "pass@4": 0.6995019520129422, + "pass@8": 0.7776365616412141, + "pass@16": 0.8346225468513689, + "pass@32": 0.8761886815828364, + "pass@64": 0.9078665970466824, + "pass@128": 0.9290123456790124 + }, + "14": { + "pass@1": 0.42811109050445106, + "pass@2": 0.543148277413024, + "pass@4": 0.6509376348357051, + "pass@8": 0.7436043737795209, + "pass@16": 0.8145172014899712, + "pass@32": 0.8642210760480362, + "pass@64": 0.9006000656184018, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.3851996527777778, + "pass@2": 0.5078033865558473, + "pass@4": 0.6108883527869196, + "pass@8": 0.6910061074300264, + "pass@16": 0.7513617520911909, + "pass@32": 0.7985095076620196, + "pass@64": 0.8389313592242905, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.26636246312684364, + "pass@2": 0.36768960641534865, + "pass@4": 0.46721277124651445, + "pass@8": 0.5581714943434668, + "pass@16": 0.6403607605318539, + "pass@32": 0.7137040695583681, + "pass@64": 0.7760761955425775, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.23943429969879518, + "pass@2": 0.33819468326060154, + "pass@4": 0.436030319534606, + "pass@8": 0.5232784476635474, + "pass@16": 0.5976341299105322, + "pass@32": 0.6617823167109694, + "pass@64": 0.7208114445140381, + "pass@128": 0.7771084337349398 + }, + "18": { + "pass@1": 0.24377520161290323, + "pass@2": 0.33654551371602726, + "pass@4": 0.4276333723405545, + "pass@8": 0.5083686482219439, + "pass@16": 0.5763213834382153, + "pass@32": 0.6355223760273493, + "pass@64": 0.6899469138467323, + "pass@128": 0.7387096774193549 + }, + "19": { + "pass@1": 0.19172488670694865, + "pass@2": 0.2705886332278707, + "pass@4": 0.35226705498670674, + "pass@8": 0.43071165135970435, + "pass@16": 0.5033697376743476, + "pass@32": 0.5710949315109166, + "pass@64": 0.6372663372318448, + "pass@128": 0.6978851963746223 + }, + "20": { + "pass@1": 0.19054992492492492, + "pass@2": 0.2684314481804639, + "pass@4": 0.35020230269639707, + "pass@8": 0.43047924091649253, + "pass@16": 0.5052137590570093, + "pass@32": 0.5743518955753166, + "pass@64": 0.639696972628713, + "pass@128": 0.6996996996996997 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6335777138157894, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6335777138157894, + "pass@2": 0.7044544589204002, + "pass@4": 0.7609624857221504, + "pass@8": 0.8055802949607513, + "pass@16": 0.8406621429211865, + "pass@32": 0.8689154536686422, + "pass@64": 0.8925336057312936, + "pass@128": 0.9123157894736842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8259609375, + "pass@2": 0.893026451771654, + "pass@4": 0.9322231326396698, + "pass@8": 0.9565009551603775, + "pass@16": 0.971507385154414, + "pass@32": 0.9808318511743037, + "pass@64": 0.9876695059302211, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.935625, + "pass@2": 0.9512100147637794, + "pass@4": 0.9636213038995125, + "pass@8": 0.9727229765191304, + "pass@16": 0.978889970330367, + "pass@32": 0.982836458697959, + "pass@64": 0.9851793710344321, + "pass@128": 0.987 + }, + "3": { + "pass@1": 0.977046875, + "pass@2": 0.9854843750000004, + "pass@4": 0.9901338926696664, + "pass@8": 0.9932631652553967, + "pass@16": 0.9954292364313113, + "pass@32": 0.9967808357169232, + "pass@64": 0.9977499040291772, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.97296875, + "pass@2": 0.9835835383858268, + "pass@4": 0.9883603146794149, + "pass@8": 0.9911434994665775, + "pass@16": 0.9930824160789711, + "pass@32": 0.9947548746152566, + "pass@64": 0.9964802260461338, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9220625, + "pass@2": 0.9477059547244095, + "pass@4": 0.963529338582677, + "pass@8": 0.9743781813113109, + "pass@16": 0.9822554465747628, + "pass@32": 0.9885125209347927, + "pass@64": 0.9929442929712965, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.9371796875, + "pass@2": 0.9675015994094499, + "pass@4": 0.9815036453880766, + "pass@8": 0.9885290906061706, + "pass@16": 0.9921437299044742, + "pass@32": 0.99416612649408, + "pass@64": 0.995942199982249, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.908703125, + "pass@2": 0.9471446850393703, + "pass@4": 0.967179319928759, + "pass@8": 0.9779200100902848, + "pass@16": 0.9847850762647822, + "pass@32": 0.9895726876030048, + "pass@64": 0.9928923860328146, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.8838984375, + "pass@2": 0.9365091043307101, + "pass@4": 0.9626962175665545, + "pass@8": 0.9774632890377969, + "pass@16": 0.9864115667605743, + "pass@32": 0.9920013654389653, + "pass@64": 0.9956127630881643, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.8575, + "pass@2": 0.9211126968503948, + "pass@4": 0.9518551809148847, + "pass@8": 0.9696605210813218, + "pass@16": 0.9819000425192099, + "pass@32": 0.9902488112271479, + "pass@64": 0.9952551317449962, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.7578203125, + "pass@2": 0.847205954724411, + "pass@4": 0.9011952808398942, + "pass@8": 0.93377580565178, + "pass@16": 0.955070876530586, + "pass@32": 0.969590082180953, + "pass@64": 0.9787324423322356, + "pass@128": 0.983 + }, + "12": { + "pass@1": 0.641609375, + "pass@2": 0.7585755413385825, + "pass@4": 0.8381425514623176, + "pass@8": 0.8892629624537244, + "pass@16": 0.9220148867551953, + "pass@32": 0.9435080487950828, + "pass@64": 0.9590211028900316, + "pass@128": 0.972 + }, + "13": { + "pass@1": 0.465140625, + "pass@2": 0.5931075295275604, + "pass@4": 0.6977997386576684, + "pass@8": 0.7762012373108262, + "pass@16": 0.8331259756536721, + "pass@32": 0.8759206686558857, + "pass@64": 0.9090539128333244, + "pass@128": 0.932 + }, + "14": { + "pass@1": 0.43965625, + "pass@2": 0.5558090551181101, + "pass@4": 0.657769242407198, + "pass@8": 0.7426246540339614, + "pass@16": 0.8089162040581582, + "pass@32": 0.8580974144957311, + "pass@64": 0.8936584566878355, + "pass@128": 0.919 + }, + "15": { + "pass@1": 0.3664609375, + "pass@2": 0.49034571850393777, + "pass@4": 0.598005425571804, + "pass@8": 0.6826972018891915, + "pass@16": 0.7454130715416146, + "pass@32": 0.7926700168179653, + "pass@64": 0.8309058005394662, + "pass@128": 0.862 + }, + "16": { + "pass@1": 0.281734375, + "pass@2": 0.3871161417322831, + "pass@4": 0.4866833546119234, + "pass@8": 0.5734005358977272, + "pass@16": 0.6484476727811473, + "pass@32": 0.7138604538727543, + "pass@64": 0.7698033716622331, + "pass@128": 0.817 + }, + "17": { + "pass@1": 0.2525, + "pass@2": 0.35274815452755964, + "pass@4": 0.4507468760779903, + "pass@8": 0.5374077389510419, + "pass@16": 0.6103234449582415, + "pass@32": 0.6721658021677973, + "pass@64": 0.7278754450284157, + "pass@128": 0.781 + }, + "18": { + "pass@1": 0.2393671875, + "pass@2": 0.337944881889764, + "pass@4": 0.435632553618297, + "pass@8": 0.5206967651312548, + "pass@16": 0.5906179695379268, + "pass@32": 0.6484050092386298, + "pass@64": 0.6969525325161366, + "pass@128": 0.736 + }, + "19": { + "pass@1": 0.188734375, + "pass@2": 0.2653257874015747, + "pass@4": 0.3448389823772027, + "pass@8": 0.42249623713425216, + "pass@16": 0.4957875266004969, + "pass@32": 0.5655016837307463, + "pass@64": 0.6324997305839606, + "pass@128": 0.695 + }, + "20": { + "pass@1": 0.1840078125, + "pass@2": 0.2631775344488189, + "pass@4": 0.34637087682789613, + "pass@8": 0.4258807772719833, + "pass@16": 0.49645821706625787, + "pass@32": 0.5599689078458404, + "pass@64": 0.6199099329613685, + "pass@128": 0.675 + } + }, + "per_op_accuracy": { + "10": 0.8259609375, + "2": 0.935625, + "3": 0.977046875, + "4": 0.97296875, + "5": 0.9220625, + "6": 0.9371796875, + "7": 0.908703125, + "8": 0.8838984375, + "9": 0.8575, + "11": 0.7578203125, + "12": 0.641609375, + "13": 0.465140625, + "14": 0.43965625, + "15": 0.3664609375, + "16": 0.281734375, + "17": 0.2525, + "18": 0.2393671875, + "19": 0.188734375, + "20": 0.1840078125 + }, + "per_op_avg_response_len": { + "10": 281.24259375, + "2": 122.4647578125, + "3": 146.4731015625, + "4": 160.6047265625, + "5": 176.30828125, + "6": 201.5447421875, + "7": 226.3348359375, + "8": 246.195390625, + "9": 262.26378125, + "11": 298.0571640625, + "12": 304.7676015625, + "13": 319.7301328125, + "14": 341.958609375, + "15": 356.6907265625, + "16": 359.9821875, + "17": 375.317109375, + "18": 383.491953125, + "19": 383.327109375, + "20": 396.21040625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.24259375, + "2": 122.4647578125, + "3": 146.4731015625, + "4": 160.6047265625, + "5": 176.30828125, + "6": 201.5447421875, + "7": 226.3348359375, + "8": 246.195390625, + "9": 262.26378125, + "11": 298.0571640625, + "12": 304.7676015625, + "13": 319.7301328125, + "14": 341.958609375, + "15": 356.6907265625, + "16": 359.9821875, + "17": 375.317109375, + "18": 383.491953125, + "19": 383.327109375, + "20": 396.21040625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 510499, + "answer_accuracy": 0.6304573881599747, + "avg_response_len": 266.6437346862156, + "resp_tokens_sum": 215908898, + "pass_at_k": { + "pass@1": 0.6304573881599747, + "pass@2": 0.7038732633227223, + "pass@4": 0.7610112612928939, + "pass@8": 0.8048920516478486, + "pass@16": 0.838512105529119, + "pass@32": 0.8650410497814764, + "pass@64": 0.8871377219105482, + "pass@128": 0.9062598798608915 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8337020648967551, + "pass@2": 0.8992248652823267, + "pass@4": 0.9340549527547986, + "pass@8": 0.9549133337554659, + "pass@16": 0.9676459667453204, + "pass@32": 0.9754629221119615, + "pass@64": 0.9816283664853948, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9328514408099688, + "pass@2": 0.9459657934113376, + "pass@4": 0.9572665329801058, + "pass@8": 0.9661779608434554, + "pass@16": 0.9726776857540342, + "pass@32": 0.9771842431316051, + "pass@64": 0.9796170198954688, + "pass@128": 0.9813084112149533 + }, + "3": { + "pass@1": 0.9756068425076453, + "pass@2": 0.9874613222085766, + "pass@4": 0.9940523552675177, + "pass@8": 0.9974856871737064, + "pass@16": 0.9991656932084267, + "pass@32": 0.9998836449467822, + "pass@64": 0.9999990650219919, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.975341796875, + "pass@2": 0.9855330339566928, + "pass@4": 0.9891930282152229, + "pass@8": 0.9909968147034551, + "pass@16": 0.9920335522380169, + "pass@32": 0.9932232658217899, + "pass@64": 0.9949311023622034, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9057291666666667, + "pass@2": 0.9370816929133859, + "pass@4": 0.9557188610352269, + "pass@8": 0.9680085338015769, + "pass@16": 0.9772068251757668, + "pass@32": 0.9850956818037647, + "pass@64": 0.9907377001622719, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9124719311377245, + "pass@2": 0.9472194881889765, + "pass@4": 0.967283268951411, + "pass@8": 0.9804620871830513, + "pass@16": 0.9883802999024801, + "pass@32": 0.9926513268166495, + "pass@64": 0.9960828629239575, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8930981878698225, + "pass@2": 0.9425206314354944, + "pass@4": 0.9694562882857096, + "pass@8": 0.9828592027889445, + "pass@16": 0.9902744534062847, + "pass@32": 0.9950889757930396, + "pass@64": 0.9980455562464515, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8852413862179487, + "pass@2": 0.9402193115283668, + "pass@4": 0.9669937225394901, + "pass@8": 0.9816570277298262, + "pass@16": 0.990420934947623, + "pass@32": 0.996331498679127, + "pass@64": 0.9995570396284617, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8653389084507043, + "pass@2": 0.9311758345347674, + "pass@4": 0.9623128128350147, + "pass@8": 0.9789012564778983, + "pass@16": 0.9887982028366319, + "pass@32": 0.994471708460012, + "pass@64": 0.9977497769018057, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.7575690509915014, + "pass@2": 0.8498137449532688, + "pass@4": 0.9047521838629934, + "pass@8": 0.9377886732347469, + "pass@16": 0.960041149186458, + "pass@32": 0.9746239590826914, + "pass@64": 0.9825700799894423, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6333200918079096, + "pass@2": 0.7547513095555853, + "pass@4": 0.8386714274592795, + "pass@8": 0.8928341163275466, + "pass@16": 0.9259803364931931, + "pass@32": 0.9447383494093016, + "pass@64": 0.9572942446713538, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.4841300940438871, + "pass@2": 0.6205060721249966, + "pass@4": 0.7295860708438091, + "pass@8": 0.8059492526033689, + "pass@16": 0.8568992077900991, + "pass@32": 0.8937514227248922, + "pass@64": 0.9225152134466567, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.4497663551401869, + "pass@2": 0.571845027841146, + "pass@4": 0.670868385610677, + "pass@8": 0.7470698554728201, + "pass@16": 0.8054958898697235, + "pass@32": 0.8505787819273606, + "pass@64": 0.884048299310893, + "pass@128": 0.9065420560747663 + }, + "15": { + "pass@1": 0.36836163294797686, + "pass@2": 0.4893602646670612, + "pass@4": 0.5954076710425649, + "pass@8": 0.6802021581752167, + "pass@16": 0.7428461601629132, + "pass@32": 0.7885083115987753, + "pass@64": 0.8230281644538102, + "pass@128": 0.8497109826589595 + }, + "16": { + "pass@1": 0.28364415322580644, + "pass@2": 0.39256453200406405, + "pass@4": 0.4944607647834345, + "pass@8": 0.5805687140274347, + "pass@16": 0.6532026734319762, + "pass@32": 0.714863241409248, + "pass@64": 0.7669355763349044, + "pass@128": 0.8129032258064516 + }, + "17": { + "pass@1": 0.2501128973988439, + "pass@2": 0.34497284773109993, + "pass@4": 0.43463790797826557, + "pass@8": 0.5132459972107867, + "pass@16": 0.5819257752825826, + "pass@32": 0.643364780253834, + "pass@64": 0.7006827171900452, + "pass@128": 0.7543352601156069 + }, + "18": { + "pass@1": 0.23118303571428572, + "pass@2": 0.32596245781777267, + "pass@4": 0.41872846885210785, + "pass@8": 0.4994272862212975, + "pass@16": 0.5665476115510434, + "pass@32": 0.6217479887245179, + "pass@64": 0.6670019576727322, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18700731981981983, + "pass@2": 0.2625891147052565, + "pass@4": 0.34095073307278057, + "pass@8": 0.4179914747194478, + "pass@16": 0.48919832053560813, + "pass@32": 0.5569396106072487, + "pass@64": 0.6262402386445067, + "pass@128": 0.7027027027027027 + }, + "20": { + "pass@1": 0.19280217717717718, + "pass@2": 0.27425383060225567, + "pass@4": 0.3585156604861329, + "pass@8": 0.43630993945564095, + "pass@16": 0.5004092983315865, + "pass@32": 0.5534055957447018, + "pass@64": 0.6021505971017288, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 522241, + "answer_accuracy": 0.6360105709275137, + "avg_response_len": 277.60607828332036, + "resp_tokens_sum": 227947903, + "pass_at_k": { + "pass@1": 0.6360105709275137, + "pass@2": 0.7061336235201687, + "pass@4": 0.7623702717255768, + "pass@8": 0.8071611606699909, + "pass@16": 0.8427868598710215, + "pass@32": 0.8718723024536624, + "pass@64": 0.8957240571987134, + "pass@128": 0.9147310989867498 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8397478911042945, + "pass@2": 0.9017401031351142, + "pass@4": 0.9367013483935669, + "pass@8": 0.9597981710002264, + "pass@16": 0.9760430566524732, + "pass@32": 0.9868893358013864, + "pass@64": 0.9939010380542732, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9402308130699089, + "pass@2": 0.9559446275997415, + "pass@4": 0.9671450020115263, + "pass@8": 0.9747791046336394, + "pass@16": 0.9798526018916596, + "pass@32": 0.982735766252235, + "pass@64": 0.9843437813422433, + "pass@128": 0.9848024316109423 + }, + "3": { + "pass@1": 0.98203125, + "pass@2": 0.9875126759723218, + "pass@4": 0.9907982465260023, + "pass@8": 0.9931905013110562, + "pass@16": 0.9947669019933483, + "pass@32": 0.996016072122554, + "pass@64": 0.9977332350126279, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9693047337278107, + "pass@2": 0.9827249947584216, + "pass@4": 0.9890199579342521, + "pass@8": 0.9924316167169746, + "pass@16": 0.9952105268225401, + "pass@32": 0.9977446704990522, + "pass@64": 0.9995571159589419, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.948052536231884, + "pass@2": 0.966407052379322, + "pass@4": 0.9765803837020368, + "pass@8": 0.9844381946054411, + "pass@16": 0.9908400253696799, + "pass@32": 0.9954288349662425, + "pass@64": 0.9982494068365969, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9469914040114613, + "pass@2": 0.9771242498251473, + "pass@4": 0.9882326420336424, + "pass@8": 0.991931583238474, + "pass@16": 0.9933659205077896, + "pass@32": 0.9940735913681014, + "pass@64": 0.9942652013230373, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9118722904624278, + "pass@2": 0.9468681728187156, + "pass@4": 0.9655806095047366, + "pass@8": 0.9770071107892204, + "pass@16": 0.9847477489910627, + "pass@32": 0.9900681636945455, + "pass@64": 0.9936873526856657, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.8875, + "pass@2": 0.9402195159143837, + "pass@4": 0.9639932780409486, + "pass@8": 0.9756221536661233, + "pass@16": 0.9821909911786181, + "pass@32": 0.987287230265571, + "pass@64": 0.992438504371287, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.8411302395209581, + "pass@2": 0.905336104366071, + "pass@4": 0.938773440464403, + "pass@8": 0.9598987045631255, + "pass@16": 0.9751816882329573, + "pass@32": 0.9858225193162448, + "pass@64": 0.9916186385525599, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.7506353539156626, + "pass@2": 0.8404700247841763, + "pass@4": 0.8961459129544045, + "pass@8": 0.9278766949955667, + "pass@16": 0.9467320736664068, + "pass@32": 0.9597594455810305, + "pass@64": 0.9681259167521782, + "pass@128": 0.9728915662650602 + }, + "12": { + "pass@1": 0.6611726097178683, + "pass@2": 0.7715347542764048, + "pass@4": 0.8428349935881834, + "pass@8": 0.8867303652307457, + "pass@16": 0.9159569131757227, + "pass@32": 0.9389731417905509, + "pass@64": 0.9577601020248315, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.44421831232493, + "pass@2": 0.5660448234411872, + "pass@4": 0.66785195968151, + "pass@8": 0.7483170301361325, + "pass@16": 0.8105249949265751, + "pass@32": 0.8597446273776105, + "pass@64": 0.8981030316546639, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.4415433114035088, + "pass@2": 0.5532334260026704, + "pass@4": 0.652206162826138, + "pass@8": 0.737487007203177, + "pass@16": 0.8066073874498777, + "pass@32": 0.8591202423066083, + "pass@64": 0.8958383932620966, + "pass@128": 0.9210526315789473 + }, + "15": { + "pass@1": 0.34607007575757576, + "pass@2": 0.4742386960152705, + "pass@4": 0.5880804396609509, + "pass@8": 0.6771553828886003, + "pass@16": 0.7422639165081902, + "pass@32": 0.7913001834008943, + "pass@64": 0.8312857431811411, + "pass@128": 0.8636363636363636 + }, + "16": { + "pass@1": 0.2948940527065527, + "pass@2": 0.4010665533795455, + "pass@4": 0.49861933924926044, + "pass@8": 0.5817780568854317, + "pass@16": 0.6520585361736072, + "pass@32": 0.7131258389048448, + "pass@64": 0.7662778139871184, + "pass@128": 0.811965811965812 + }, + "17": { + "pass@1": 0.2685364906832298, + "pass@2": 0.37610842972074143, + "pass@4": 0.48323010506838776, + "pass@8": 0.577938445005628, + "pass@16": 0.653921104286253, + "pass@32": 0.7138194380494706, + "pass@64": 0.7643783394471984, + "pass@128": 0.8136645962732919 + }, + "18": { + "pass@1": 0.24377297794117647, + "pass@2": 0.3515556247105141, + "pass@4": 0.46032718851320026, + "pass@8": 0.5538321588382297, + "pass@16": 0.6284314018506372, + "pass@32": 0.6875919900487388, + "pass@64": 0.7341714824652724, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.1875, + "pass@2": 0.2628534958520808, + "pass@4": 0.34137497935525907, + "pass@8": 0.41886761741849504, + "pass@16": 0.4948485199164589, + "pass@32": 0.5684772948464838, + "pass@64": 0.6340076592011885, + "pass@128": 0.6845238095238095 + }, + "20": { + "pass@1": 0.16871725299401197, + "pass@2": 0.24689621740770412, + "pass@4": 0.3304425002638143, + "pass@8": 0.41089814433551575, + "pass@16": 0.483789637592383, + "pass@32": 0.5521726444199829, + "pass@64": 0.6178883001231452, + "pass@128": 0.6766467065868264 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 508121, + "answer_accuracy": 0.6342379473558076, + "avg_response_len": 299.6219768533312, + "resp_tokens_sum": 240042746, + "pass_at_k": { + "pass@1": 0.6342379473558076, + "pass@2": 0.703320859694323, + "pass@4": 0.7594703142135963, + "pass@8": 0.8046556382540034, + "pass@16": 0.8406575219448124, + "pass@32": 0.8698007858357516, + "pass@64": 0.8947172796227344, + "pass@128": 0.9159610161367631 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8047108208955224, + "pass@2": 0.8782744740862615, + "pass@4": 0.9260115345656411, + "pass@8": 0.9548988993170184, + "pass@16": 0.9710010924150604, + "pass@32": 0.980370140618196, + "pass@64": 0.9877186605551607, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9338392857142858, + "pass@2": 0.9515691788526434, + "pass@4": 0.9661372604317318, + "pass@8": 0.9767929304683264, + "pass@16": 0.9836826490884742, + "pass@32": 0.9881149987306532, + "pass@64": 0.9910662959611092, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9736242711370262, + "pass@2": 0.9816482203117467, + "pass@4": 0.9857590412350061, + "pass@8": 0.9893075222068398, + "pass@16": 0.9925043063975238, + "pass@32": 0.9945585423291038, + "pass@64": 0.9956216974134651, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9743695175438597, + "pass@2": 0.9826079511442651, + "pass@4": 0.9869292393823577, + "pass@8": 0.9900076968161828, + "pass@16": 0.9919605885870972, + "pass@32": 0.9932331313555678, + "pass@64": 0.9948887956026504, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9108226102941176, + "pass@2": 0.9385729070171378, + "pass@4": 0.9575225440569924, + "pass@8": 0.9700715177204019, + "pass@16": 0.9782220232113423, + "pass@32": 0.9846600973625091, + "pass@64": 0.9896055060633964, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9524102917981072, + "pass@2": 0.9782773727613706, + "pass@4": 0.9890783643172356, + "pass@8": 0.9932827474347146, + "pass@16": 0.994763417917439, + "pass@32": 0.9958640376966922, + "pass@64": 0.9976402791921362, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9219244462025317, + "pass@2": 0.9523934298066383, + "pass@4": 0.9664943151884493, + "pass@8": 0.973636516501745, + "pass@16": 0.9789543982992107, + "pass@32": 0.983129839071662, + "pass@64": 0.9865100126337769, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.8788006756756757, + "pass@2": 0.9290773302830391, + "pass@4": 0.9572869742633514, + "pass@8": 0.9754967922961485, + "pass@16": 0.9871544540195394, + "pass@32": 0.992969883203604, + "pass@64": 0.9953011942110427, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.8661324356913184, + "pass@2": 0.9265691856343519, + "pass@4": 0.9539672130011069, + "pass@8": 0.9695961662623275, + "pass@16": 0.9812411467601234, + "pass@32": 0.9901821005537559, + "pass@64": 0.9963129764254025, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.7656746031746032, + "pass@2": 0.8513830302462192, + "pass@4": 0.9025311645568108, + "pass@8": 0.9354963214647195, + "pass@16": 0.9582898489222895, + "pass@32": 0.974310122831304, + "pass@64": 0.9856107928071083, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.6314984709480123, + "pass@2": 0.7500733674059092, + "pass@4": 0.8329923644865482, + "pass@8": 0.887867580321604, + "pass@16": 0.923631780836543, + "pass@32": 0.946600124993897, + "pass@64": 0.9621206964233943, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.4694974922839506, + "pass@2": 0.5959508967629046, + "pass@4": 0.6995019520129422, + "pass@8": 0.7776365616412141, + "pass@16": 0.8346225468513689, + "pass@32": 0.8761886815828364, + "pass@64": 0.9078665970466824, + "pass@128": 0.9290123456790124 + }, + "14": { + "pass@1": 0.42811109050445106, + "pass@2": 0.543148277413024, + "pass@4": 0.6509376348357051, + "pass@8": 0.7436043737795209, + "pass@16": 0.8145172014899712, + "pass@32": 0.8642210760480362, + "pass@64": 0.9006000656184018, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.3851996527777778, + "pass@2": 0.5078033865558473, + "pass@4": 0.6108883527869196, + "pass@8": 0.6910061074300264, + "pass@16": 0.7513617520911909, + "pass@32": 0.7985095076620196, + "pass@64": 0.8389313592242905, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.26636246312684364, + "pass@2": 0.36768960641534865, + "pass@4": 0.46721277124651445, + "pass@8": 0.5581714943434668, + "pass@16": 0.6403607605318539, + "pass@32": 0.7137040695583681, + "pass@64": 0.7760761955425775, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.23943429969879518, + "pass@2": 0.33819468326060154, + "pass@4": 0.436030319534606, + "pass@8": 0.5232784476635474, + "pass@16": 0.5976341299105322, + "pass@32": 0.6617823167109694, + "pass@64": 0.7208114445140381, + "pass@128": 0.7771084337349398 + }, + "18": { + "pass@1": 0.24377520161290323, + "pass@2": 0.33654551371602726, + "pass@4": 0.4276333723405545, + "pass@8": 0.5083686482219439, + "pass@16": 0.5763213834382153, + "pass@32": 0.6355223760273493, + "pass@64": 0.6899469138467323, + "pass@128": 0.7387096774193549 + }, + "19": { + "pass@1": 0.19172488670694865, + "pass@2": 0.2705886332278707, + "pass@4": 0.35226705498670674, + "pass@8": 0.43071165135970435, + "pass@16": 0.5033697376743476, + "pass@32": 0.5710949315109166, + "pass@64": 0.6372663372318448, + "pass@128": 0.6978851963746223 + }, + "20": { + "pass@1": 0.19054992492492492, + "pass@2": 0.2684314481804639, + "pass@4": 0.35020230269639707, + "pass@8": 0.43047924091649253, + "pass@16": 0.5052137590570093, + "pass@32": 0.5743518955753166, + "pass@64": 0.639696972628713, + "pass@128": 0.6996996996996997 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..313d4e60ad8a0c40f94d3212c2040c07962d5e31 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e22b3eb97fe3857e6aa68e2b8d4f34312b580d5dfd860e8e8d52a60ab38ca4e5 +size 2642913151 diff --git a/results/composition/SDAR-100M-block16-epoch1-op17-20/summary.csv b/results/composition/SDAR-100M-block16-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..238def96b32e34ef264d76dccd03d776c75a1bcf --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op17-20,0.9134383680555556,0.381703125,0.6335777138157894,0.0,0.0,0.0,202.60357899305555,351.9533,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch1-op17-20/summary.json b/results/composition/SDAR-100M-block16-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..56ce1e019ef6817e7a8b50e917747a80a9c970ff --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op17-20", + "id_acc": 0.9134383680555556, + "ood_acc": 0.381703125, + "total_acc": 0.6335777138157894, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 202.60357899305555, + "ood_avg_resp_len": 351.9533, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 510499, + "answer_accuracy": 0.6304573881599747, + "avg_response_len": 266.6437346862156, + "resp_tokens_sum": 215908898, + "pass_at_k": { + "pass@1": 0.6304573881599747, + "pass@2": 0.7038732633227223, + "pass@4": 0.7610112612928939, + "pass@8": 0.8048920516478486, + "pass@16": 0.838512105529119, + "pass@32": 0.8650410497814764, + "pass@64": 0.8871377219105482, + "pass@128": 0.9062598798608915 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8337020648967551, + "pass@2": 0.8992248652823267, + "pass@4": 0.9340549527547986, + "pass@8": 0.9549133337554659, + "pass@16": 0.9676459667453204, + "pass@32": 0.9754629221119615, + "pass@64": 0.9816283664853948, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9328514408099688, + "pass@2": 0.9459657934113376, + "pass@4": 0.9572665329801058, + "pass@8": 0.9661779608434554, + "pass@16": 0.9726776857540342, + "pass@32": 0.9771842431316051, + "pass@64": 0.9796170198954688, + "pass@128": 0.9813084112149533 + }, + "3": { + "pass@1": 0.9756068425076453, + "pass@2": 0.9874613222085766, + "pass@4": 0.9940523552675177, + "pass@8": 0.9974856871737064, + "pass@16": 0.9991656932084267, + "pass@32": 0.9998836449467822, + "pass@64": 0.9999990650219919, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.975341796875, + "pass@2": 0.9855330339566928, + "pass@4": 0.9891930282152229, + "pass@8": 0.9909968147034551, + "pass@16": 0.9920335522380169, + "pass@32": 0.9932232658217899, + "pass@64": 0.9949311023622034, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9057291666666667, + "pass@2": 0.9370816929133859, + "pass@4": 0.9557188610352269, + "pass@8": 0.9680085338015769, + "pass@16": 0.9772068251757668, + "pass@32": 0.9850956818037647, + "pass@64": 0.9907377001622719, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9124719311377245, + "pass@2": 0.9472194881889765, + "pass@4": 0.967283268951411, + "pass@8": 0.9804620871830513, + "pass@16": 0.9883802999024801, + "pass@32": 0.9926513268166495, + "pass@64": 0.9960828629239575, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8930981878698225, + "pass@2": 0.9425206314354944, + "pass@4": 0.9694562882857096, + "pass@8": 0.9828592027889445, + "pass@16": 0.9902744534062847, + "pass@32": 0.9950889757930396, + "pass@64": 0.9980455562464515, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8852413862179487, + "pass@2": 0.9402193115283668, + "pass@4": 0.9669937225394901, + "pass@8": 0.9816570277298262, + "pass@16": 0.990420934947623, + "pass@32": 0.996331498679127, + "pass@64": 0.9995570396284617, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8653389084507043, + "pass@2": 0.9311758345347674, + "pass@4": 0.9623128128350147, + "pass@8": 0.9789012564778983, + "pass@16": 0.9887982028366319, + "pass@32": 0.994471708460012, + "pass@64": 0.9977497769018057, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.7575690509915014, + "pass@2": 0.8498137449532688, + "pass@4": 0.9047521838629934, + "pass@8": 0.9377886732347469, + "pass@16": 0.960041149186458, + "pass@32": 0.9746239590826914, + "pass@64": 0.9825700799894423, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6333200918079096, + "pass@2": 0.7547513095555853, + "pass@4": 0.8386714274592795, + "pass@8": 0.8928341163275466, + "pass@16": 0.9259803364931931, + "pass@32": 0.9447383494093016, + "pass@64": 0.9572942446713538, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.4841300940438871, + "pass@2": 0.6205060721249966, + "pass@4": 0.7295860708438091, + "pass@8": 0.8059492526033689, + "pass@16": 0.8568992077900991, + "pass@32": 0.8937514227248922, + "pass@64": 0.9225152134466567, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.4497663551401869, + "pass@2": 0.571845027841146, + "pass@4": 0.670868385610677, + "pass@8": 0.7470698554728201, + "pass@16": 0.8054958898697235, + "pass@32": 0.8505787819273606, + "pass@64": 0.884048299310893, + "pass@128": 0.9065420560747663 + }, + "15": { + "pass@1": 0.36836163294797686, + "pass@2": 0.4893602646670612, + "pass@4": 0.5954076710425649, + "pass@8": 0.6802021581752167, + "pass@16": 0.7428461601629132, + "pass@32": 0.7885083115987753, + "pass@64": 0.8230281644538102, + "pass@128": 0.8497109826589595 + }, + "16": { + "pass@1": 0.28364415322580644, + "pass@2": 0.39256453200406405, + "pass@4": 0.4944607647834345, + "pass@8": 0.5805687140274347, + "pass@16": 0.6532026734319762, + "pass@32": 0.714863241409248, + "pass@64": 0.7669355763349044, + "pass@128": 0.8129032258064516 + }, + "17": { + "pass@1": 0.2501128973988439, + "pass@2": 0.34497284773109993, + "pass@4": 0.43463790797826557, + "pass@8": 0.5132459972107867, + "pass@16": 0.5819257752825826, + "pass@32": 0.643364780253834, + "pass@64": 0.7006827171900452, + "pass@128": 0.7543352601156069 + }, + "18": { + "pass@1": 0.23118303571428572, + "pass@2": 0.32596245781777267, + "pass@4": 0.41872846885210785, + "pass@8": 0.4994272862212975, + "pass@16": 0.5665476115510434, + "pass@32": 0.6217479887245179, + "pass@64": 0.6670019576727322, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18700731981981983, + "pass@2": 0.2625891147052565, + "pass@4": 0.34095073307278057, + "pass@8": 0.4179914747194478, + "pass@16": 0.48919832053560813, + "pass@32": 0.5569396106072487, + "pass@64": 0.6262402386445067, + "pass@128": 0.7027027027027027 + }, + "20": { + "pass@1": 0.19280217717717718, + "pass@2": 0.27425383060225567, + "pass@4": 0.3585156604861329, + "pass@8": 0.43630993945564095, + "pass@16": 0.5004092983315865, + "pass@32": 0.5534055957447018, + "pass@64": 0.6021505971017288, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 522241, + "answer_accuracy": 0.6360105709275137, + "avg_response_len": 277.60607828332036, + "resp_tokens_sum": 227947903, + "pass_at_k": { + "pass@1": 0.6360105709275137, + "pass@2": 0.7061336235201687, + "pass@4": 0.7623702717255768, + "pass@8": 0.8071611606699909, + "pass@16": 0.8427868598710215, + "pass@32": 0.8718723024536624, + "pass@64": 0.8957240571987134, + "pass@128": 0.9147310989867498 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8397478911042945, + "pass@2": 0.9017401031351142, + "pass@4": 0.9367013483935669, + "pass@8": 0.9597981710002264, + "pass@16": 0.9760430566524732, + "pass@32": 0.9868893358013864, + "pass@64": 0.9939010380542732, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9402308130699089, + "pass@2": 0.9559446275997415, + "pass@4": 0.9671450020115263, + "pass@8": 0.9747791046336394, + "pass@16": 0.9798526018916596, + "pass@32": 0.982735766252235, + "pass@64": 0.9843437813422433, + "pass@128": 0.9848024316109423 + }, + "3": { + "pass@1": 0.98203125, + "pass@2": 0.9875126759723218, + "pass@4": 0.9907982465260023, + "pass@8": 0.9931905013110562, + "pass@16": 0.9947669019933483, + "pass@32": 0.996016072122554, + "pass@64": 0.9977332350126279, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9693047337278107, + "pass@2": 0.9827249947584216, + "pass@4": 0.9890199579342521, + "pass@8": 0.9924316167169746, + "pass@16": 0.9952105268225401, + "pass@32": 0.9977446704990522, + "pass@64": 0.9995571159589419, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.948052536231884, + "pass@2": 0.966407052379322, + "pass@4": 0.9765803837020368, + "pass@8": 0.9844381946054411, + "pass@16": 0.9908400253696799, + "pass@32": 0.9954288349662425, + "pass@64": 0.9982494068365969, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9469914040114613, + "pass@2": 0.9771242498251473, + "pass@4": 0.9882326420336424, + "pass@8": 0.991931583238474, + "pass@16": 0.9933659205077896, + "pass@32": 0.9940735913681014, + "pass@64": 0.9942652013230373, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9118722904624278, + "pass@2": 0.9468681728187156, + "pass@4": 0.9655806095047366, + "pass@8": 0.9770071107892204, + "pass@16": 0.9847477489910627, + "pass@32": 0.9900681636945455, + "pass@64": 0.9936873526856657, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.8875, + "pass@2": 0.9402195159143837, + "pass@4": 0.9639932780409486, + "pass@8": 0.9756221536661233, + "pass@16": 0.9821909911786181, + "pass@32": 0.987287230265571, + "pass@64": 0.992438504371287, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.8411302395209581, + "pass@2": 0.905336104366071, + "pass@4": 0.938773440464403, + "pass@8": 0.9598987045631255, + "pass@16": 0.9751816882329573, + "pass@32": 0.9858225193162448, + "pass@64": 0.9916186385525599, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.7506353539156626, + "pass@2": 0.8404700247841763, + "pass@4": 0.8961459129544045, + "pass@8": 0.9278766949955667, + "pass@16": 0.9467320736664068, + "pass@32": 0.9597594455810305, + "pass@64": 0.9681259167521782, + "pass@128": 0.9728915662650602 + }, + "12": { + "pass@1": 0.6611726097178683, + "pass@2": 0.7715347542764048, + "pass@4": 0.8428349935881834, + "pass@8": 0.8867303652307457, + "pass@16": 0.9159569131757227, + "pass@32": 0.9389731417905509, + "pass@64": 0.9577601020248315, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.44421831232493, + "pass@2": 0.5660448234411872, + "pass@4": 0.66785195968151, + "pass@8": 0.7483170301361325, + "pass@16": 0.8105249949265751, + "pass@32": 0.8597446273776105, + "pass@64": 0.8981030316546639, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.4415433114035088, + "pass@2": 0.5532334260026704, + "pass@4": 0.652206162826138, + "pass@8": 0.737487007203177, + "pass@16": 0.8066073874498777, + "pass@32": 0.8591202423066083, + "pass@64": 0.8958383932620966, + "pass@128": 0.9210526315789473 + }, + "15": { + "pass@1": 0.34607007575757576, + "pass@2": 0.4742386960152705, + "pass@4": 0.5880804396609509, + "pass@8": 0.6771553828886003, + "pass@16": 0.7422639165081902, + "pass@32": 0.7913001834008943, + "pass@64": 0.8312857431811411, + "pass@128": 0.8636363636363636 + }, + "16": { + "pass@1": 0.2948940527065527, + "pass@2": 0.4010665533795455, + "pass@4": 0.49861933924926044, + "pass@8": 0.5817780568854317, + "pass@16": 0.6520585361736072, + "pass@32": 0.7131258389048448, + "pass@64": 0.7662778139871184, + "pass@128": 0.811965811965812 + }, + "17": { + "pass@1": 0.2685364906832298, + "pass@2": 0.37610842972074143, + "pass@4": 0.48323010506838776, + "pass@8": 0.577938445005628, + "pass@16": 0.653921104286253, + "pass@32": 0.7138194380494706, + "pass@64": 0.7643783394471984, + "pass@128": 0.8136645962732919 + }, + "18": { + "pass@1": 0.24377297794117647, + "pass@2": 0.3515556247105141, + "pass@4": 0.46032718851320026, + "pass@8": 0.5538321588382297, + "pass@16": 0.6284314018506372, + "pass@32": 0.6875919900487388, + "pass@64": 0.7341714824652724, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.1875, + "pass@2": 0.2628534958520808, + "pass@4": 0.34137497935525907, + "pass@8": 0.41886761741849504, + "pass@16": 0.4948485199164589, + "pass@32": 0.5684772948464838, + "pass@64": 0.6340076592011885, + "pass@128": 0.6845238095238095 + }, + "20": { + "pass@1": 0.16871725299401197, + "pass@2": 0.24689621740770412, + "pass@4": 0.3304425002638143, + "pass@8": 0.41089814433551575, + "pass@16": 0.483789637592383, + "pass@32": 0.5521726444199829, + "pass@64": 0.6178883001231452, + "pass@128": 0.6766467065868264 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 508121, + "answer_accuracy": 0.6342379473558076, + "avg_response_len": 299.6219768533312, + "resp_tokens_sum": 240042746, + "pass_at_k": { + "pass@1": 0.6342379473558076, + "pass@2": 0.703320859694323, + "pass@4": 0.7594703142135963, + "pass@8": 0.8046556382540034, + "pass@16": 0.8406575219448124, + "pass@32": 0.8698007858357516, + "pass@64": 0.8947172796227344, + "pass@128": 0.9159610161367631 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8047108208955224, + "pass@2": 0.8782744740862615, + "pass@4": 0.9260115345656411, + "pass@8": 0.9548988993170184, + "pass@16": 0.9710010924150604, + "pass@32": 0.980370140618196, + "pass@64": 0.9877186605551607, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9338392857142858, + "pass@2": 0.9515691788526434, + "pass@4": 0.9661372604317318, + "pass@8": 0.9767929304683264, + "pass@16": 0.9836826490884742, + "pass@32": 0.9881149987306532, + "pass@64": 0.9910662959611092, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9736242711370262, + "pass@2": 0.9816482203117467, + "pass@4": 0.9857590412350061, + "pass@8": 0.9893075222068398, + "pass@16": 0.9925043063975238, + "pass@32": 0.9945585423291038, + "pass@64": 0.9956216974134651, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9743695175438597, + "pass@2": 0.9826079511442651, + "pass@4": 0.9869292393823577, + "pass@8": 0.9900076968161828, + "pass@16": 0.9919605885870972, + "pass@32": 0.9932331313555678, + "pass@64": 0.9948887956026504, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9108226102941176, + "pass@2": 0.9385729070171378, + "pass@4": 0.9575225440569924, + "pass@8": 0.9700715177204019, + "pass@16": 0.9782220232113423, + "pass@32": 0.9846600973625091, + "pass@64": 0.9896055060633964, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9524102917981072, + "pass@2": 0.9782773727613706, + "pass@4": 0.9890783643172356, + "pass@8": 0.9932827474347146, + "pass@16": 0.994763417917439, + "pass@32": 0.9958640376966922, + "pass@64": 0.9976402791921362, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9219244462025317, + "pass@2": 0.9523934298066383, + "pass@4": 0.9664943151884493, + "pass@8": 0.973636516501745, + "pass@16": 0.9789543982992107, + "pass@32": 0.983129839071662, + "pass@64": 0.9865100126337769, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.8788006756756757, + "pass@2": 0.9290773302830391, + "pass@4": 0.9572869742633514, + "pass@8": 0.9754967922961485, + "pass@16": 0.9871544540195394, + "pass@32": 0.992969883203604, + "pass@64": 0.9953011942110427, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.8661324356913184, + "pass@2": 0.9265691856343519, + "pass@4": 0.9539672130011069, + "pass@8": 0.9695961662623275, + "pass@16": 0.9812411467601234, + "pass@32": 0.9901821005537559, + "pass@64": 0.9963129764254025, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.7656746031746032, + "pass@2": 0.8513830302462192, + "pass@4": 0.9025311645568108, + "pass@8": 0.9354963214647195, + "pass@16": 0.9582898489222895, + "pass@32": 0.974310122831304, + "pass@64": 0.9856107928071083, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.6314984709480123, + "pass@2": 0.7500733674059092, + "pass@4": 0.8329923644865482, + "pass@8": 0.887867580321604, + "pass@16": 0.923631780836543, + "pass@32": 0.946600124993897, + "pass@64": 0.9621206964233943, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.4694974922839506, + "pass@2": 0.5959508967629046, + "pass@4": 0.6995019520129422, + "pass@8": 0.7776365616412141, + "pass@16": 0.8346225468513689, + "pass@32": 0.8761886815828364, + "pass@64": 0.9078665970466824, + "pass@128": 0.9290123456790124 + }, + "14": { + "pass@1": 0.42811109050445106, + "pass@2": 0.543148277413024, + "pass@4": 0.6509376348357051, + "pass@8": 0.7436043737795209, + "pass@16": 0.8145172014899712, + "pass@32": 0.8642210760480362, + "pass@64": 0.9006000656184018, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.3851996527777778, + "pass@2": 0.5078033865558473, + "pass@4": 0.6108883527869196, + "pass@8": 0.6910061074300264, + "pass@16": 0.7513617520911909, + "pass@32": 0.7985095076620196, + "pass@64": 0.8389313592242905, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.26636246312684364, + "pass@2": 0.36768960641534865, + "pass@4": 0.46721277124651445, + "pass@8": 0.5581714943434668, + "pass@16": 0.6403607605318539, + "pass@32": 0.7137040695583681, + "pass@64": 0.7760761955425775, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.23943429969879518, + "pass@2": 0.33819468326060154, + "pass@4": 0.436030319534606, + "pass@8": 0.5232784476635474, + "pass@16": 0.5976341299105322, + "pass@32": 0.6617823167109694, + "pass@64": 0.7208114445140381, + "pass@128": 0.7771084337349398 + }, + "18": { + "pass@1": 0.24377520161290323, + "pass@2": 0.33654551371602726, + "pass@4": 0.4276333723405545, + "pass@8": 0.5083686482219439, + "pass@16": 0.5763213834382153, + "pass@32": 0.6355223760273493, + "pass@64": 0.6899469138467323, + "pass@128": 0.7387096774193549 + }, + "19": { + "pass@1": 0.19172488670694865, + "pass@2": 0.2705886332278707, + "pass@4": 0.35226705498670674, + "pass@8": 0.43071165135970435, + "pass@16": 0.5033697376743476, + "pass@32": 0.5710949315109166, + "pass@64": 0.6372663372318448, + "pass@128": 0.6978851963746223 + }, + "20": { + "pass@1": 0.19054992492492492, + "pass@2": 0.2684314481804639, + "pass@4": 0.35020230269639707, + "pass@8": 0.43047924091649253, + "pass@16": 0.5052137590570093, + "pass@32": 0.5743518955753166, + "pass@64": 0.639696972628713, + "pass@128": 0.6996996996996997 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..df3b227b49fc1c12c2c1684efb029d4c7cbcac03 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dee1f707cf113068b69edb919f48014cf0f1b22e99ed2c0f70166c454a0f4302 +size 1791146605 diff --git a/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json b/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4c13e85573399810b91d58749e035d315b0d3783 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9702256944444444, + "avg_response_len": 199.40936458333334, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9702256944444444, + "pass@2": 0.9793916502624618, + "pass@4": 0.9849351835395576, + "pass@8": 0.9888442521480617, + "pass@16": 0.9918560073630215, + "pass@32": 0.9942361825582353, + "pass@64": 0.9961668747436009, + "pass@128": 0.9978888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9150703125, + "pass@2": 0.9398096702755907, + "pass@4": 0.9555400461192349, + "pass@8": 0.9668906410889534, + "pass@16": 0.9758948266659789, + "pass@32": 0.9832636557874399, + "pass@64": 0.9892967113568902, + "pass@128": 0.995 + }, + "2": { + "pass@1": 0.990578125, + "pass@2": 0.9930145177165355, + "pass@4": 0.9943968228346454, + "pass@8": 0.9952873162624573, + "pass@16": 0.9960729357688821, + "pass@32": 0.996952502557328, + "pass@64": 0.9979116198042409, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.99659375, + "pass@2": 0.9988639271653545, + "pass@4": 0.9997500349643794, + "pass@8": 0.9999798240813113, + "pass@16": 0.9999998412707761, + "pass@32": 0.9999999999968697, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9929296875, + "pass@2": 0.9952704232283465, + "pass@4": 0.9970510909261342, + "pass@8": 0.9984842712829364, + "pass@16": 0.9994335154423146, + "pass@32": 0.999865478267178, + "pass@64": 0.9999934228152456, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.962078125, + "pass@2": 0.9698677411417324, + "pass@4": 0.9763977542182228, + "pass@8": 0.9819934096902104, + "pass@16": 0.9865521659361441, + "pass@32": 0.9903191098925549, + "pass@64": 0.9939691902252709, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9701875, + "pass@2": 0.9778256643700791, + "pass@4": 0.9825182044431944, + "pass@8": 0.9861330184093109, + "pass@16": 0.9892783904066073, + "pass@32": 0.9920173554056341, + "pass@64": 0.9943151204054765, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.97875, + "pass@2": 0.9858042568897639, + "pass@4": 0.9899331335770527, + "pass@8": 0.9927339385848097, + "pass@16": 0.9948275081708874, + "pass@32": 0.9963086569131329, + "pass@64": 0.9973199896660145, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.964484375, + "pass@2": 0.977889886811024, + "pass@4": 0.9856384359767532, + "pass@8": 0.9905634974996607, + "pass@16": 0.9937330000695483, + "pass@32": 0.9958854298336186, + "pass@64": 0.9975687158504051, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.961359375, + "pass@2": 0.97617876476378, + "pass@4": 0.9831911287964004, + "pass@8": 0.9875323524329049, + "pass@16": 0.9909118825360496, + "pass@32": 0.9935134543703622, + "pass@64": 0.9951271025688606, + "pass@128": 0.996 + } + }, + "per_op_accuracy": { + "10": 0.9150703125, + "2": 0.990578125, + "3": 0.99659375, + "4": 0.9929296875, + "5": 0.962078125, + "6": 0.9701875, + "7": 0.97875, + "8": 0.964484375, + "9": 0.961359375 + }, + "per_op_avg_response_len": { + "10": 278.345546875, + "2": 116.3494296875, + "3": 144.50065625, + "4": 159.5111484375, + "5": 173.8181953125, + "6": 198.9171640625, + "7": 223.07409375, + "8": 242.34471875, + "9": 257.823328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.345546875, + "2": 116.3494296875, + "3": 144.50065625, + "4": 159.5111484375, + "5": 173.8181953125, + "6": 198.9171640625, + "7": 223.07409375, + "8": 242.34471875, + "9": 257.823328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 366904, + "answer_accuracy": 0.9680639986491051, + "avg_response_len": 189.9049070204323, + "resp_tokens_sum": 71975479, + "pass_at_k": { + "pass@1": 0.9680639986491051, + "pass@2": 0.9787410217339862, + "pass@4": 0.9848598221701518, + "pass@8": 0.9887697662104195, + "pass@16": 0.9918066653339469, + "pass@32": 0.9943563851377097, + "pass@64": 0.9963836343066652, + "pass@128": 0.9979736575481256 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9222437315634219, + "pass@2": 0.9451468248437975, + "pass@4": 0.9604848572578869, + "pass@8": 0.9712668741187709, + "pass@16": 0.9797398811861905, + "pass@32": 0.986931871980946, + "pass@64": 0.9931403532607942, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.989242601246106, + "pass@2": 0.9923153224421711, + "pass@4": 0.9941436283665476, + "pass@8": 0.9953145921083346, + "pass@16": 0.9966062996683891, + "pass@32": 0.9983101878563488, + "pass@64": 0.9997246792749038, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975630733944955, + "pass@2": 0.9996982530280046, + "pass@4": 0.9999930091880717, + "pass@8": 0.9999999967989345, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992236328125, + "pass@2": 0.9945377860482283, + "pass@4": 0.9961152392083802, + "pass@8": 0.9975110009296438, + "pass@16": 0.9987271357419916, + "pass@32": 0.9996050186206877, + "pass@64": 0.99997946273747, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9551587301587302, + "pass@2": 0.963551431071116, + "pass@4": 0.9705333979085945, + "pass@8": 0.9768221559126241, + "pass@16": 0.9826882032901565, + "pass@32": 0.988084362473456, + "pass@64": 0.992762368619646, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9563061377245509, + "pass@2": 0.9692498614974779, + "pass@4": 0.9767356600073196, + "pass@8": 0.9813628882947097, + "pass@16": 0.9850560727092349, + "pass@32": 0.9883559011032365, + "pass@64": 0.991386843357507, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9775563979289941, + "pass@2": 0.9875821908633461, + "pass@4": 0.9925620808123836, + "pass@8": 0.9950344430852842, + "pass@16": 0.9964798475945668, + "pass@32": 0.997005436160734, + "pass@64": 0.9970413979979968, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9700270432692307, + "pass@2": 0.9840835888602868, + "pass@4": 0.9896981008383567, + "pass@8": 0.9919003031750214, + "pass@16": 0.9935285960287868, + "pass@32": 0.9953138505545496, + "pass@64": 0.9966101352715523, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9554577464788733, + "pass@2": 0.9742978540534544, + "pass@4": 0.984426338080979, + "pass@8": 0.9901960111237609, + "pass@16": 0.9936839096661296, + "pass@32": 0.9957886523598386, + "pass@64": 0.9969763514894339, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 380695, + "answer_accuracy": 0.9745018635321101, + "avg_response_len": 197.31111770969855, + "resp_tokens_sum": 77080772, + "pass_at_k": { + "pass@1": 0.9745018635321101, + "pass@2": 0.9822816908494244, + "pass@4": 0.986926477581931, + "pass@8": 0.9903302991720578, + "pass@16": 0.9929998545725408, + "pass@32": 0.9951115215182154, + "pass@64": 0.9968114142410952, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9157400306748467, + "pass@2": 0.9375864239650259, + "pass@4": 0.9503189637721664, + "pass@8": 0.959476709600869, + "pass@16": 0.9678207783989702, + "pass@32": 0.976828113172386, + "pass@64": 0.9855899073216104, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9930661094224924, + "pass@2": 0.9945559497881914, + "pass@4": 0.9952527896398966, + "pass@8": 0.9960218094796784, + "pass@16": 0.996694876664963, + "pass@32": 0.9969450828883747, + "pass@64": 0.9969604795045498, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9994791666666667, + "pass@2": 0.9999657003101885, + "pass@4": 0.9999997156605425, + "pass@8": 0.9999999999936351, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9902921597633136, + "pass@2": 0.9935943542375252, + "pass@4": 0.996376267685474, + "pass@8": 0.9984966632337737, + "pass@16": 0.9996459679388677, + "pass@32": 0.9999785901153704, + "pass@64": 0.9999999845150909, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9790760869565217, + "pass@2": 0.9860735478717333, + "pass@4": 0.990809667269852, + "pass@8": 0.9943024238372346, + "pass@16": 0.9967581848072867, + "pass@32": 0.9982186985410357, + "pass@64": 0.9992791299089842, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9788905802292264, + "pass@2": 0.9836452716422623, + "pass@4": 0.9860912927610409, + "pass@8": 0.9885082645987024, + "pass@16": 0.9914576644920206, + "pass@32": 0.994320594090395, + "pass@64": 0.9964244742548366, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.979091401734104, + "pass@2": 0.9841676954849575, + "pass@4": 0.9884633377655827, + "pass@8": 0.9922804183734506, + "pass@16": 0.9951181931156187, + "pass@32": 0.9970528714118317, + "pass@64": 0.9984707380171003, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9671654929577465, + "pass@2": 0.9811702894532549, + "pass@4": 0.9894243719535057, + "pass@8": 0.9944174851855382, + "pass@16": 0.9969106947639146, + "pass@32": 0.9978444559613356, + "pass@64": 0.9985914715905386, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9666448353293413, + "pass@2": 0.9787814872224057, + "pass@4": 0.984590454074977, + "pass@8": 0.9884570828941244, + "pass@16": 0.9916722775023806, + "pass@32": 0.9940966746741055, + "pass@64": 0.9954845654245074, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 370101, + "answer_accuracy": 0.9679993513558754, + "avg_response_len": 210.97499843069969, + "resp_tokens_sum": 80663337, + "pass_at_k": { + "pass@1": 0.9679993513558754, + "pass@2": 0.9770836848126664, + "pass@4": 0.9829752624138431, + "pass@8": 0.9873997048913209, + "pass@16": 0.9907361814723746, + "pass@32": 0.9932226390886398, + "pass@64": 0.9952934366409588, + "pass@128": 0.9973217274857716 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9071595149253732, + "pass@2": 0.9365723057938655, + "pass@4": 0.9556170069226421, + "pass@8": 0.9696769057695721, + "pass@16": 0.9798609945844656, + "pass@32": 0.9858142874379154, + "pass@64": 0.9890143934765259, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9894642857142857, + "pass@2": 0.99220683352081, + "pass@4": 0.9938244295355938, + "pass@8": 0.9945718767910509, + "pass@16": 0.9949991404358748, + "pass@32": 0.9957142856718992, + "pass@64": 0.9971428571428571, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9928935860058309, + "pass@2": 0.9970085053143868, + "pass@4": 0.9992781772358628, + "pass@8": 0.9999411811374931, + "pass@16": 0.9999995372325835, + "pass@32": 0.9999999999908739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9961851242690059, + "pass@2": 0.9976123975457015, + "pass@4": 0.998593672227375, + "pass@8": 0.9993826865860662, + "pass@16": 0.99988448784076, + "pass@32": 0.9999973942969671, + "pass@64": 0.9999999999215047, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9512408088235295, + "pass@2": 0.9592754892311257, + "pass@4": 0.9672070549085778, + "pass@8": 0.9742943951584943, + "pass@16": 0.9797759062977376, + "pass@32": 0.9843737785787031, + "pass@64": 0.9896992479161258, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9752316640378549, + "pass@2": 0.9804542928289322, + "pass@4": 0.9846770751645001, + "pass@8": 0.9885439412426841, + "pass@16": 0.9913278776467133, + "pass@32": 0.9933394230271457, + "pass@64": 0.9950781489247041, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9796528876582279, + "pass@2": 0.985694473861258, + "pass@4": 0.9887304917581505, + "pass@8": 0.9907698483062334, + "pass@16": 0.9927418508415174, + "pass@32": 0.994748499944022, + "pass@64": 0.996357980344224, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9564329954954955, + "pass@2": 0.9685896526447708, + "pass@4": 0.9777987281432166, + "pass@8": 0.9852023893939588, + "pass@16": 0.9905368812834143, + "pass@32": 0.9943325122952712, + "pass@64": 0.9973765202133318, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.96241961414791, + "pass@2": 0.9755305782717674, + "pass@4": 0.9802783508572683, + "pass@8": 0.9834987228210043, + "pass@16": 0.9869310415394815, + "pass@32": 0.9902900110658145, + "pass@64": 0.9926323245605343, + "pass@128": 0.9935691318327974 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.35165546875, + "avg_response_len": 320.09001796875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.35165546875, + "pass@2": 0.4197094857283516, + "pass@4": 0.4786157665635541, + "pass@8": 0.5292358452246045, + "pass@16": 0.5726114897631275, + "pass@32": 0.609898000220599, + "pass@64": 0.6423396090115875, + "pass@128": 0.6712 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8648359375, + "pass@2": 0.9100951033464573, + "pass@4": 0.9347501322647173, + "pass@8": 0.9492648414627204, + "pass@16": 0.9591873478095747, + "pass@32": 0.9663478688104976, + "pass@64": 0.9718211052876301, + "pass@128": 0.976 + }, + "12": { + "pass@1": 0.7038046875, + "pass@2": 0.7893309547244096, + "pass@4": 0.847530710161229, + "pass@8": 0.8872501406974933, + "pass@16": 0.913835808415057, + "pass@32": 0.9321118322142374, + "pass@64": 0.9463118241321323, + "pass@128": 0.957 + }, + "13": { + "pass@1": 0.4401484375, + "pass@2": 0.5271272145669298, + "pass@4": 0.6008414419760031, + "pass@8": 0.663756701110009, + "pass@16": 0.716035068325726, + "pass@32": 0.7594926715885558, + "pass@64": 0.796119320741206, + "pass@128": 0.829 + }, + "14": { + "pass@1": 0.3608828125, + "pass@2": 0.4397880167322835, + "pass@4": 0.508594947975253, + "pass@8": 0.5686336976073794, + "pass@16": 0.6218801756842585, + "pass@32": 0.6686538789478669, + "pass@64": 0.7076235209573943, + "pass@128": 0.741 + }, + "15": { + "pass@1": 0.2531953125, + "pass@2": 0.32912573818897634, + "pass@4": 0.40056516357330324, + "pass@8": 0.4651769969975074, + "pass@16": 0.5209395590458854, + "pass@32": 0.5685441656287817, + "pass@64": 0.6091709442131789, + "pass@128": 0.644 + }, + "16": { + "pass@1": 0.21353125, + "pass@2": 0.277843626968504, + "pass@4": 0.3358475869891258, + "pass@8": 0.38845783283534113, + "pass@16": 0.4373589747108126, + "pass@32": 0.481591610081772, + "pass@64": 0.5213278775966319, + "pass@128": 0.557 + }, + "17": { + "pass@1": 0.1928125, + "pass@2": 0.25692937992125997, + "pass@4": 0.3162516365766777, + "pass@8": 0.3696114083029441, + "pass@16": 0.4174144617061861, + "pass@32": 0.46007529401338054, + "pass@64": 0.5000393438805595, + "pass@128": 0.543 + }, + "18": { + "pass@1": 0.17340625, + "pass@2": 0.23288939468503944, + "pass@4": 0.2924173040869889, + "pass@8": 0.34748396702279544, + "pass@16": 0.39519400799025706, + "pass@32": 0.4368171730593396, + "pass@64": 0.47577441414294835, + "pass@128": 0.512 + }, + "19": { + "pass@1": 0.152296875, + "pass@2": 0.21116178641732283, + "pass@4": 0.26833084833145837, + "pass@8": 0.32048557651113246, + "pass@16": 0.3667058127901986, + "pass@32": 0.4060712287023584, + "pass@64": 0.43877021147199496, + "pass@128": 0.467 + }, + "20": { + "pass@1": 0.161640625, + "pass@2": 0.22280364173228356, + "pass@4": 0.2810278937007869, + "pass@8": 0.3322372896986801, + "pass@16": 0.37756368115318545, + "pass@32": 0.41927427915921184, + "pass@64": 0.45643752769216683, + "pass@128": 0.486 + } + }, + "per_op_accuracy": { + "11": 0.8648359375, + "12": 0.7038046875, + "13": 0.4401484375, + "14": 0.3608828125, + "15": 0.2531953125, + "16": 0.21353125, + "17": 0.1928125, + "18": 0.17340625, + "19": 0.152296875, + "20": 0.161640625 + }, + "per_op_avg_response_len": { + "11": 294.4192421875, + "12": 301.11725, + "13": 310.0360859375, + "14": 320.811046875, + "15": 323.3763125, + "16": 327.6389453125, + "17": 333.5490078125, + "18": 329.8962734375, + "19": 329.6489375, + "20": 330.407078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 294.4192421875, + "12": 301.11725, + "13": 310.0360859375, + "14": 320.811046875, + "15": 323.3763125, + "16": 327.6389453125, + "17": 333.5490078125, + "18": 329.8962734375, + "19": 329.6489375, + "20": 330.407078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 152549, + "answer_accuracy": 0.3541720839524517, + "avg_response_len": 303.4558088781575, + "resp_tokens_sum": 130704486, + "pass_at_k": { + "pass@1": 0.3541720839524517, + "pass@2": 0.42192966035263496, + "pass@4": 0.4798859520160286, + "pass@8": 0.5286846055548091, + "pass@16": 0.5698062594651296, + "pass@32": 0.6048244550283293, + "pass@64": 0.6348565285474133, + "pass@128": 0.6612184249628529 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8513411827195467, + "pass@2": 0.9002692054605079, + "pass@4": 0.9301656318278912, + "pass@8": 0.9489189290719003, + "pass@16": 0.9620749038932601, + "pass@32": 0.9706335966755882, + "pass@64": 0.9751169549682261, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.7018449858757062, + "pass@2": 0.786281779661017, + "pass@4": 0.8425637121101385, + "pass@8": 0.8814481070980112, + "pass@16": 0.9089087677573177, + "pass@32": 0.9300411469087887, + "pass@64": 0.9476799728882689, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.46071708463949845, + "pass@2": 0.5479957822180537, + "pass@4": 0.6168102769481397, + "pass@8": 0.6711818283871478, + "pass@16": 0.714081813122014, + "pass@32": 0.7497040752868626, + "pass@64": 0.7797252476653084, + "pass@128": 0.8056426332288401 + }, + "14": { + "pass@1": 0.3388093847352025, + "pass@2": 0.41182321178894693, + "pass@4": 0.4764041463275034, + "pass@8": 0.534130178473544, + "pass@16": 0.5871949465142853, + "pass@32": 0.6363122142046183, + "pass@64": 0.6797314428750697, + "pass@128": 0.7165109034267912 + }, + "15": { + "pass@1": 0.26275740606936415, + "pass@2": 0.3398819751035454, + "pass@4": 0.41254214727855526, + "pass@8": 0.4781416760702823, + "pass@16": 0.5331817011339766, + "pass@32": 0.5794008648263183, + "pass@64": 0.6195336061079341, + "pass@128": 0.6560693641618497 + }, + "16": { + "pass@1": 0.19909274193548387, + "pass@2": 0.26266430657861306, + "pass@4": 0.3225466130241786, + "pass@8": 0.37652892333302973, + "pass@16": 0.4252326950906888, + "pass@32": 0.46738340757399127, + "pass@64": 0.505813596566932, + "pass@128": 0.5451612903225806 + }, + "17": { + "pass@1": 0.18987084537572255, + "pass@2": 0.25301889422420454, + "pass@4": 0.30949784781237605, + "pass@8": 0.35793822819914667, + "pass@16": 0.39987978151086667, + "pass@32": 0.4364144577614477, + "pass@64": 0.46841234177173346, + "pass@128": 0.5028901734104047 + }, + "18": { + "pass@1": 0.16642857142857143, + "pass@2": 0.22563835770528673, + "pass@4": 0.2839633877015374, + "pass@8": 0.3352756128571621, + "pass@16": 0.3794013307508793, + "pass@32": 0.41912972908665974, + "pass@64": 0.45480493264252136, + "pass@128": 0.4857142857142857 + }, + "19": { + "pass@1": 0.15287162162162163, + "pass@2": 0.21274547184980255, + "pass@4": 0.2720780853969829, + "pass@8": 0.3269558100570385, + "pass@16": 0.3740083610670051, + "pass@32": 0.41088179701472477, + "pass@64": 0.44192531549939784, + "pass@128": 0.47147147147147145 + }, + "20": { + "pass@1": 0.17898367117117117, + "pass@2": 0.24102498167458797, + "pass@4": 0.2964189808481148, + "pass@8": 0.34303163036602097, + "pass@16": 0.38364439504923087, + "pass@32": 0.4206433730102397, + "pass@64": 0.4507408603585201, + "pass@128": 0.46846846846846846 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 149088, + "answer_accuracy": 0.34634255129348795, + "avg_response_len": 315.6940580396967, + "resp_tokens_sum": 135894927, + "pass_at_k": { + "pass@1": 0.34634255129348795, + "pass@2": 0.4140275441289073, + "pass@4": 0.4736084765720087, + "pass@8": 0.5251538742682398, + "pass@16": 0.5689560116969068, + "pass@32": 0.6061521589618866, + "pass@64": 0.6385964343815986, + "pass@128": 0.6687481415402914 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.867211031626506, + "pass@2": 0.911044522578503, + "pass@4": 0.9335228710116054, + "pass@8": 0.9456478076783712, + "pass@16": 0.9524671555395998, + "pass@32": 0.9570610201613988, + "pass@64": 0.9621282660404785, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.71608052507837, + "pass@2": 0.7989217967072296, + "pass@4": 0.8551101025185328, + "pass@8": 0.8918423946053244, + "pass@16": 0.9147090576639251, + "pass@32": 0.9288851299053897, + "pass@64": 0.9409932385294371, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.40434611344537813, + "pass@2": 0.48740185050398094, + "pass@4": 0.5640730444408734, + "pass@8": 0.6355232978685984, + "pass@16": 0.6996316769009948, + "pass@32": 0.7527484934731532, + "pass@64": 0.7943710656136685, + "pass@128": 0.8291316526610645 + }, + "14": { + "pass@1": 0.3604486476608187, + "pass@2": 0.44008114322880676, + "pass@4": 0.5081644520312152, + "pass@8": 0.5657154899960264, + "pass@16": 0.6164087304791761, + "pass@32": 0.6613545821335254, + "pass@64": 0.6976279435761811, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.22407670454545456, + "pass@2": 0.30122323132903833, + "pass@4": 0.37661032541386885, + "pass@8": 0.44410643751587464, + "pass@16": 0.4993273457645693, + "pass@32": 0.5427207028934755, + "pass@64": 0.5788425426868631, + "pass@128": 0.6121212121212121 + }, + "16": { + "pass@1": 0.23039084757834757, + "pass@2": 0.2978291074993831, + "pass@4": 0.35624147569160675, + "pass@8": 0.40731401792660654, + "pass@16": 0.45254311760056604, + "pass@32": 0.49147289445246195, + "pass@64": 0.5235553530937181, + "pass@128": 0.5470085470085471 + }, + "17": { + "pass@1": 0.21370341614906832, + "pass@2": 0.2806520363623024, + "pass@4": 0.3440671235971277, + "pass@8": 0.40228302586205095, + "pass@16": 0.4537379226594085, + "pass@32": 0.500026224710816, + "pass@64": 0.5454292278608023, + "pass@128": 0.593167701863354 + }, + "18": { + "pass@1": 0.17212775735294117, + "pass@2": 0.23196720125057887, + "pass@4": 0.2933669291338581, + "pass@8": 0.35235556676313684, + "pass@16": 0.4034011348916748, + "pass@32": 0.4468507861160033, + "pass@64": 0.48950582673115856, + "pass@128": 0.5352941176470588 + }, + "19": { + "pass@1": 0.15087890625, + "pass@2": 0.2072612017247844, + "pass@4": 0.2605428718731588, + "pass@8": 0.3074317250275432, + "pass@16": 0.34908958061974404, + "pass@32": 0.3879378881426714, + "pass@64": 0.42367879660082997, + "pass@128": 0.45535714285714285 + }, + "20": { + "pass@1": 0.1435254491017964, + "pass@2": 0.20275848401621951, + "pass@4": 0.26188721965517786, + "pass@8": 0.31475370332114117, + "pass@16": 0.3611758484951675, + "pass@32": 0.40307471926923255, + "pass@64": 0.43896682078335747, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 148482, + "answer_accuracy": 0.35452800275061125, + "avg_response_len": 341.7152401054401, + "resp_tokens_sum": 143115810, + "pass_at_k": { + "pass@1": 0.35452800275061125, + "pass@2": 0.42326617337754974, + "pass@4": 0.4824560300702804, + "pass@8": 0.5339982504248119, + "pass@16": 0.5792535963307709, + "pass@32": 0.6189657703077097, + "pass@64": 0.6538826597580932, + "pass@128": 0.6839853300733496 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8774553571428572, + "pass@2": 0.9201056899137611, + "pass@4": 0.9411811779479943, + "pass@8": 0.9534647217495896, + "pass@16": 0.9630343209400213, + "pass@32": 0.9713331761600971, + "pass@64": 0.9783436058997058, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.6939506880733946, + "pass@2": 0.7832756928893064, + "pass@4": 0.8455138635193533, + "pass@8": 0.8890513361030553, + "pass@16": 0.9183177835907471, + "pass@32": 0.9375012531153086, + "pass@64": 0.9500191762654265, + "pass@128": 0.9571865443425076 + }, + "13": { + "pass@1": 0.4593460648148148, + "pass@2": 0.550352158671138, + "pass@4": 0.6256323665560319, + "pass@8": 0.6875551867759847, + "pass@16": 0.7360322879202137, + "pass@32": 0.7765612944510213, + "pass@64": 0.8141867170120144, + "pass@128": 0.8518518518518519 + }, + "14": { + "pass@1": 0.38234885014836795, + "pass@2": 0.46612764024860376, + "pass@4": 0.5396942860814502, + "pass@8": 0.6044605719250148, + "pass@16": 0.6604712523124485, + "pass@32": 0.7068676293724594, + "pass@64": 0.7443352257906309, + "pass@128": 0.7744807121661721 + }, + "15": { + "pass@1": 0.2726417824074074, + "pass@2": 0.3460583594585398, + "pass@4": 0.41217337416156297, + "pass@8": 0.4727927552375029, + "pass@16": 0.5298785998494511, + "pass@32": 0.5832519581605827, + "pass@64": 0.6289946833739761, + "pass@128": 0.6635802469135802 + }, + "16": { + "pass@1": 0.20927820796460178, + "pass@2": 0.2710314902561958, + "pass@4": 0.32689492325627434, + "pass@8": 0.37984261448337203, + "pass@16": 0.43272626830354155, + "pass@32": 0.48435329728914633, + "pass@64": 0.5332086540560117, + "pass@128": 0.5781710914454278 + }, + "17": { + "pass@1": 0.17561652861445784, + "pass@2": 0.2379966470211554, + "pass@4": 0.29631255251602584, + "pass@8": 0.3500892982483715, + "pass@16": 0.4004591753225813, + "pass@32": 0.44598628696095505, + "pass@64": 0.4889772959528953, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18268649193548386, + "pass@2": 0.24208748729997448, + "pass@4": 0.3009205241481911, + "pass@8": 0.35592454781716937, + "pass@16": 0.40402308504606516, + "pass@32": 0.44578225999860455, + "pass@64": 0.4843893762241014, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.15315804380664652, + "pass@2": 0.21352803780003343, + "pass@4": 0.27246658895734704, + "pass@8": 0.32722728747094926, + "pass@16": 0.3772414787512142, + "pass@32": 0.41963885160277065, + "pass@64": 0.4509154252350958, + "pass@128": 0.4743202416918429 + }, + "20": { + "pass@1": 0.16246715465465467, + "pass@2": 0.22468765517485983, + "pass@4": 0.284834960100708, + "pass@8": 0.33897903867127327, + "pass@16": 0.3879200126258433, + "pass@32": 0.4341533925251601, + "pass@64": 0.4796573665199342, + "pass@128": 0.5195195195195195 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6446624177631579, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6446624177631579, + "pass@2": 0.6848220899813319, + "pass@4": 0.7184512798679576, + "pass@8": 0.746945090609406, + "pass@16": 0.7712009980999249, + "pass@32": 0.7919529286963355, + "pass@64": 0.8099419980425414, + "pass@128": 0.8259473684210527 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9150703125, + "pass@2": 0.9398096702755907, + "pass@4": 0.9555400461192349, + "pass@8": 0.9668906410889534, + "pass@16": 0.9758948266659789, + "pass@32": 0.9832636557874399, + "pass@64": 0.9892967113568902, + "pass@128": 0.995 + }, + "2": { + "pass@1": 0.990578125, + "pass@2": 0.9930145177165355, + "pass@4": 0.9943968228346454, + "pass@8": 0.9952873162624573, + "pass@16": 0.9960729357688821, + "pass@32": 0.996952502557328, + "pass@64": 0.9979116198042409, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.99659375, + "pass@2": 0.9988639271653545, + "pass@4": 0.9997500349643794, + "pass@8": 0.9999798240813113, + "pass@16": 0.9999998412707761, + "pass@32": 0.9999999999968697, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9929296875, + "pass@2": 0.9952704232283465, + "pass@4": 0.9970510909261342, + "pass@8": 0.9984842712829364, + "pass@16": 0.9994335154423146, + "pass@32": 0.999865478267178, + "pass@64": 0.9999934228152456, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.962078125, + "pass@2": 0.9698677411417324, + "pass@4": 0.9763977542182228, + "pass@8": 0.9819934096902104, + "pass@16": 0.9865521659361441, + "pass@32": 0.9903191098925549, + "pass@64": 0.9939691902252709, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9701875, + "pass@2": 0.9778256643700791, + "pass@4": 0.9825182044431944, + "pass@8": 0.9861330184093109, + "pass@16": 0.9892783904066073, + "pass@32": 0.9920173554056341, + "pass@64": 0.9943151204054765, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.97875, + "pass@2": 0.9858042568897639, + "pass@4": 0.9899331335770527, + "pass@8": 0.9927339385848097, + "pass@16": 0.9948275081708874, + "pass@32": 0.9963086569131329, + "pass@64": 0.9973199896660145, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.964484375, + "pass@2": 0.977889886811024, + "pass@4": 0.9856384359767532, + "pass@8": 0.9905634974996607, + "pass@16": 0.9937330000695483, + "pass@32": 0.9958854298336186, + "pass@64": 0.9975687158504051, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.961359375, + "pass@2": 0.97617876476378, + "pass@4": 0.9831911287964004, + "pass@8": 0.9875323524329049, + "pass@16": 0.9909118825360496, + "pass@32": 0.9935134543703622, + "pass@64": 0.9951271025688606, + "pass@128": 0.996 + }, + "11": { + "pass@1": 0.8648359375, + "pass@2": 0.9100951033464573, + "pass@4": 0.9347501322647173, + "pass@8": 0.9492648414627204, + "pass@16": 0.9591873478095747, + "pass@32": 0.9663478688104976, + "pass@64": 0.9718211052876301, + "pass@128": 0.976 + }, + "12": { + "pass@1": 0.7038046875, + "pass@2": 0.7893309547244096, + "pass@4": 0.847530710161229, + "pass@8": 0.8872501406974933, + "pass@16": 0.913835808415057, + "pass@32": 0.9321118322142374, + "pass@64": 0.9463118241321323, + "pass@128": 0.957 + }, + "13": { + "pass@1": 0.4401484375, + "pass@2": 0.5271272145669298, + "pass@4": 0.6008414419760031, + "pass@8": 0.663756701110009, + "pass@16": 0.716035068325726, + "pass@32": 0.7594926715885558, + "pass@64": 0.796119320741206, + "pass@128": 0.829 + }, + "14": { + "pass@1": 0.3608828125, + "pass@2": 0.4397880167322835, + "pass@4": 0.508594947975253, + "pass@8": 0.5686336976073794, + "pass@16": 0.6218801756842585, + "pass@32": 0.6686538789478669, + "pass@64": 0.7076235209573943, + "pass@128": 0.741 + }, + "15": { + "pass@1": 0.2531953125, + "pass@2": 0.32912573818897634, + "pass@4": 0.40056516357330324, + "pass@8": 0.4651769969975074, + "pass@16": 0.5209395590458854, + "pass@32": 0.5685441656287817, + "pass@64": 0.6091709442131789, + "pass@128": 0.644 + }, + "16": { + "pass@1": 0.21353125, + "pass@2": 0.277843626968504, + "pass@4": 0.3358475869891258, + "pass@8": 0.38845783283534113, + "pass@16": 0.4373589747108126, + "pass@32": 0.481591610081772, + "pass@64": 0.5213278775966319, + "pass@128": 0.557 + }, + "17": { + "pass@1": 0.1928125, + "pass@2": 0.25692937992125997, + "pass@4": 0.3162516365766777, + "pass@8": 0.3696114083029441, + "pass@16": 0.4174144617061861, + "pass@32": 0.46007529401338054, + "pass@64": 0.5000393438805595, + "pass@128": 0.543 + }, + "18": { + "pass@1": 0.17340625, + "pass@2": 0.23288939468503944, + "pass@4": 0.2924173040869889, + "pass@8": 0.34748396702279544, + "pass@16": 0.39519400799025706, + "pass@32": 0.4368171730593396, + "pass@64": 0.47577441414294835, + "pass@128": 0.512 + }, + "19": { + "pass@1": 0.152296875, + "pass@2": 0.21116178641732283, + "pass@4": 0.26833084833145837, + "pass@8": 0.32048557651113246, + "pass@16": 0.3667058127901986, + "pass@32": 0.4060712287023584, + "pass@64": 0.43877021147199496, + "pass@128": 0.467 + }, + "20": { + "pass@1": 0.161640625, + "pass@2": 0.22280364173228356, + "pass@4": 0.2810278937007869, + "pass@8": 0.3322372896986801, + "pass@16": 0.37756368115318545, + "pass@32": 0.41927427915921184, + "pass@64": 0.45643752769216683, + "pass@128": 0.486 + } + }, + "per_op_accuracy": { + "10": 0.9150703125, + "2": 0.990578125, + "3": 0.99659375, + "4": 0.9929296875, + "5": 0.962078125, + "6": 0.9701875, + "7": 0.97875, + "8": 0.964484375, + "9": 0.961359375, + "11": 0.8648359375, + "12": 0.7038046875, + "13": 0.4401484375, + "14": 0.3608828125, + "15": 0.2531953125, + "16": 0.21353125, + "17": 0.1928125, + "18": 0.17340625, + "19": 0.152296875, + "20": 0.161640625 + }, + "per_op_avg_response_len": { + "10": 278.345546875, + "2": 116.3494296875, + "3": 144.50065625, + "4": 159.5111484375, + "5": 173.8181953125, + "6": 198.9171640625, + "7": 223.07409375, + "8": 242.34471875, + "9": 257.823328125, + "11": 294.4192421875, + "12": 301.11725, + "13": 310.0360859375, + "14": 320.811046875, + "15": 323.3763125, + "16": 327.6389453125, + "17": 333.5490078125, + "18": 329.8962734375, + "19": 329.6489375, + "20": 330.407078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.345546875, + "2": 116.3494296875, + "3": 144.50065625, + "4": 159.5111484375, + "5": 173.8181953125, + "6": 198.9171640625, + "7": 223.07409375, + "8": 242.34471875, + "9": 257.823328125, + "11": 294.4192421875, + "12": 301.11725, + "13": 310.0360859375, + "14": 320.811046875, + "15": 323.3763125, + "16": 327.6389453125, + "17": 333.5490078125, + "18": 329.8962734375, + "19": 329.6489375, + "20": 330.407078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 519453, + "answer_accuracy": 0.6415154224628518, + "avg_response_len": 250.30623246324691, + "resp_tokens_sum": 202679965, + "pass_at_k": { + "pass@1": 0.6415154224628518, + "pass@2": 0.6825554019034074, + "pass@4": 0.7162482077109957, + "pass@8": 0.7440358797725249, + "pass@16": 0.7673312676500218, + "pass@32": 0.7871519992986242, + "pass@64": 0.8040759025836397, + "pass@128": 0.8188428706923806 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9222437315634219, + "pass@2": 0.9451468248437975, + "pass@4": 0.9604848572578869, + "pass@8": 0.9712668741187709, + "pass@16": 0.9797398811861905, + "pass@32": 0.986931871980946, + "pass@64": 0.9931403532607942, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.989242601246106, + "pass@2": 0.9923153224421711, + "pass@4": 0.9941436283665476, + "pass@8": 0.9953145921083346, + "pass@16": 0.9966062996683891, + "pass@32": 0.9983101878563488, + "pass@64": 0.9997246792749038, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975630733944955, + "pass@2": 0.9996982530280046, + "pass@4": 0.9999930091880717, + "pass@8": 0.9999999967989345, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992236328125, + "pass@2": 0.9945377860482283, + "pass@4": 0.9961152392083802, + "pass@8": 0.9975110009296438, + "pass@16": 0.9987271357419916, + "pass@32": 0.9996050186206877, + "pass@64": 0.99997946273747, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9551587301587302, + "pass@2": 0.963551431071116, + "pass@4": 0.9705333979085945, + "pass@8": 0.9768221559126241, + "pass@16": 0.9826882032901565, + "pass@32": 0.988084362473456, + "pass@64": 0.992762368619646, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9563061377245509, + "pass@2": 0.9692498614974779, + "pass@4": 0.9767356600073196, + "pass@8": 0.9813628882947097, + "pass@16": 0.9850560727092349, + "pass@32": 0.9883559011032365, + "pass@64": 0.991386843357507, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9775563979289941, + "pass@2": 0.9875821908633461, + "pass@4": 0.9925620808123836, + "pass@8": 0.9950344430852842, + "pass@16": 0.9964798475945668, + "pass@32": 0.997005436160734, + "pass@64": 0.9970413979979968, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9700270432692307, + "pass@2": 0.9840835888602868, + "pass@4": 0.9896981008383567, + "pass@8": 0.9919003031750214, + "pass@16": 0.9935285960287868, + "pass@32": 0.9953138505545496, + "pass@64": 0.9966101352715523, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9554577464788733, + "pass@2": 0.9742978540534544, + "pass@4": 0.984426338080979, + "pass@8": 0.9901960111237609, + "pass@16": 0.9936839096661296, + "pass@32": 0.9957886523598386, + "pass@64": 0.9969763514894339, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8513411827195467, + "pass@2": 0.9002692054605079, + "pass@4": 0.9301656318278912, + "pass@8": 0.9489189290719003, + "pass@16": 0.9620749038932601, + "pass@32": 0.9706335966755882, + "pass@64": 0.9751169549682261, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.7018449858757062, + "pass@2": 0.786281779661017, + "pass@4": 0.8425637121101385, + "pass@8": 0.8814481070980112, + "pass@16": 0.9089087677573177, + "pass@32": 0.9300411469087887, + "pass@64": 0.9476799728882689, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.46071708463949845, + "pass@2": 0.5479957822180537, + "pass@4": 0.6168102769481397, + "pass@8": 0.6711818283871478, + "pass@16": 0.714081813122014, + "pass@32": 0.7497040752868626, + "pass@64": 0.7797252476653084, + "pass@128": 0.8056426332288401 + }, + "14": { + "pass@1": 0.3388093847352025, + "pass@2": 0.41182321178894693, + "pass@4": 0.4764041463275034, + "pass@8": 0.534130178473544, + "pass@16": 0.5871949465142853, + "pass@32": 0.6363122142046183, + "pass@64": 0.6797314428750697, + "pass@128": 0.7165109034267912 + }, + "15": { + "pass@1": 0.26275740606936415, + "pass@2": 0.3398819751035454, + "pass@4": 0.41254214727855526, + "pass@8": 0.4781416760702823, + "pass@16": 0.5331817011339766, + "pass@32": 0.5794008648263183, + "pass@64": 0.6195336061079341, + "pass@128": 0.6560693641618497 + }, + "16": { + "pass@1": 0.19909274193548387, + "pass@2": 0.26266430657861306, + "pass@4": 0.3225466130241786, + "pass@8": 0.37652892333302973, + "pass@16": 0.4252326950906888, + "pass@32": 0.46738340757399127, + "pass@64": 0.505813596566932, + "pass@128": 0.5451612903225806 + }, + "17": { + "pass@1": 0.18987084537572255, + "pass@2": 0.25301889422420454, + "pass@4": 0.30949784781237605, + "pass@8": 0.35793822819914667, + "pass@16": 0.39987978151086667, + "pass@32": 0.4364144577614477, + "pass@64": 0.46841234177173346, + "pass@128": 0.5028901734104047 + }, + "18": { + "pass@1": 0.16642857142857143, + "pass@2": 0.22563835770528673, + "pass@4": 0.2839633877015374, + "pass@8": 0.3352756128571621, + "pass@16": 0.3794013307508793, + "pass@32": 0.41912972908665974, + "pass@64": 0.45480493264252136, + "pass@128": 0.4857142857142857 + }, + "19": { + "pass@1": 0.15287162162162163, + "pass@2": 0.21274547184980255, + "pass@4": 0.2720780853969829, + "pass@8": 0.3269558100570385, + "pass@16": 0.3740083610670051, + "pass@32": 0.41088179701472477, + "pass@64": 0.44192531549939784, + "pass@128": 0.47147147147147145 + }, + "20": { + "pass@1": 0.17898367117117117, + "pass@2": 0.24102498167458797, + "pass@4": 0.2964189808481148, + "pass@8": 0.34303163036602097, + "pass@16": 0.38364439504923087, + "pass@32": 0.4206433730102397, + "pass@64": 0.4507408603585201, + "pass@128": 0.46846846846846846 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 529783, + "answer_accuracy": 0.6451955865159782, + "avg_response_len": 259.37219772992984, + "resp_tokens_sum": 212975699, + "pass_at_k": { + "pass@1": 0.6451955865159782, + "pass@2": 0.68438010153982, + "pass@4": 0.7178246167251279, + "pass@8": 0.7464661811749366, + "pass@16": 0.770699083942655, + "pass@32": 0.79120344103857, + "pass@64": 0.8090207708634686, + "pass@128": 0.8255650818394388 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9157400306748467, + "pass@2": 0.9375864239650259, + "pass@4": 0.9503189637721664, + "pass@8": 0.959476709600869, + "pass@16": 0.9678207783989702, + "pass@32": 0.976828113172386, + "pass@64": 0.9855899073216104, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9930661094224924, + "pass@2": 0.9945559497881914, + "pass@4": 0.9952527896398966, + "pass@8": 0.9960218094796784, + "pass@16": 0.996694876664963, + "pass@32": 0.9969450828883747, + "pass@64": 0.9969604795045498, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9994791666666667, + "pass@2": 0.9999657003101885, + "pass@4": 0.9999997156605425, + "pass@8": 0.9999999999936351, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9902921597633136, + "pass@2": 0.9935943542375252, + "pass@4": 0.996376267685474, + "pass@8": 0.9984966632337737, + "pass@16": 0.9996459679388677, + "pass@32": 0.9999785901153704, + "pass@64": 0.9999999845150909, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9790760869565217, + "pass@2": 0.9860735478717333, + "pass@4": 0.990809667269852, + "pass@8": 0.9943024238372346, + "pass@16": 0.9967581848072867, + "pass@32": 0.9982186985410357, + "pass@64": 0.9992791299089842, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9788905802292264, + "pass@2": 0.9836452716422623, + "pass@4": 0.9860912927610409, + "pass@8": 0.9885082645987024, + "pass@16": 0.9914576644920206, + "pass@32": 0.994320594090395, + "pass@64": 0.9964244742548366, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.979091401734104, + "pass@2": 0.9841676954849575, + "pass@4": 0.9884633377655827, + "pass@8": 0.9922804183734506, + "pass@16": 0.9951181931156187, + "pass@32": 0.9970528714118317, + "pass@64": 0.9984707380171003, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9671654929577465, + "pass@2": 0.9811702894532549, + "pass@4": 0.9894243719535057, + "pass@8": 0.9944174851855382, + "pass@16": 0.9969106947639146, + "pass@32": 0.9978444559613356, + "pass@64": 0.9985914715905386, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9666448353293413, + "pass@2": 0.9787814872224057, + "pass@4": 0.984590454074977, + "pass@8": 0.9884570828941244, + "pass@16": 0.9916722775023806, + "pass@32": 0.9940966746741055, + "pass@64": 0.9954845654245074, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.867211031626506, + "pass@2": 0.911044522578503, + "pass@4": 0.9335228710116054, + "pass@8": 0.9456478076783712, + "pass@16": 0.9524671555395998, + "pass@32": 0.9570610201613988, + "pass@64": 0.9621282660404785, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.71608052507837, + "pass@2": 0.7989217967072296, + "pass@4": 0.8551101025185328, + "pass@8": 0.8918423946053244, + "pass@16": 0.9147090576639251, + "pass@32": 0.9288851299053897, + "pass@64": 0.9409932385294371, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.40434611344537813, + "pass@2": 0.48740185050398094, + "pass@4": 0.5640730444408734, + "pass@8": 0.6355232978685984, + "pass@16": 0.6996316769009948, + "pass@32": 0.7527484934731532, + "pass@64": 0.7943710656136685, + "pass@128": 0.8291316526610645 + }, + "14": { + "pass@1": 0.3604486476608187, + "pass@2": 0.44008114322880676, + "pass@4": 0.5081644520312152, + "pass@8": 0.5657154899960264, + "pass@16": 0.6164087304791761, + "pass@32": 0.6613545821335254, + "pass@64": 0.6976279435761811, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.22407670454545456, + "pass@2": 0.30122323132903833, + "pass@4": 0.37661032541386885, + "pass@8": 0.44410643751587464, + "pass@16": 0.4993273457645693, + "pass@32": 0.5427207028934755, + "pass@64": 0.5788425426868631, + "pass@128": 0.6121212121212121 + }, + "16": { + "pass@1": 0.23039084757834757, + "pass@2": 0.2978291074993831, + "pass@4": 0.35624147569160675, + "pass@8": 0.40731401792660654, + "pass@16": 0.45254311760056604, + "pass@32": 0.49147289445246195, + "pass@64": 0.5235553530937181, + "pass@128": 0.5470085470085471 + }, + "17": { + "pass@1": 0.21370341614906832, + "pass@2": 0.2806520363623024, + "pass@4": 0.3440671235971277, + "pass@8": 0.40228302586205095, + "pass@16": 0.4537379226594085, + "pass@32": 0.500026224710816, + "pass@64": 0.5454292278608023, + "pass@128": 0.593167701863354 + }, + "18": { + "pass@1": 0.17212775735294117, + "pass@2": 0.23196720125057887, + "pass@4": 0.2933669291338581, + "pass@8": 0.35235556676313684, + "pass@16": 0.4034011348916748, + "pass@32": 0.4468507861160033, + "pass@64": 0.48950582673115856, + "pass@128": 0.5352941176470588 + }, + "19": { + "pass@1": 0.15087890625, + "pass@2": 0.2072612017247844, + "pass@4": 0.2605428718731588, + "pass@8": 0.3074317250275432, + "pass@16": 0.34908958061974404, + "pass@32": 0.3879378881426714, + "pass@64": 0.42367879660082997, + "pass@128": 0.45535714285714285 + }, + "20": { + "pass@1": 0.1435254491017964, + "pass@2": 0.20275848401621951, + "pass@4": 0.26188721965517786, + "pass@8": 0.31475370332114117, + "pass@16": 0.3611758484951675, + "pass@32": 0.40307471926923255, + "pass@64": 0.43896682078335747, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 518583, + "answer_accuracy": 0.6472966428343185, + "avg_response_len": 279.3217104868989, + "resp_tokens_sum": 223779147, + "pass_at_k": { + "pass@1": 0.6472966428343185, + "pass@2": 0.687566046625146, + "pass@4": 0.7213202171624994, + "pass@8": 0.7503762891676571, + "pass@16": 0.7756265763304547, + "pass@32": 0.7975734180227835, + "pass@64": 0.8168150755671897, + "pass@128": 0.8335197315865154 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9071595149253732, + "pass@2": 0.9365723057938655, + "pass@4": 0.9556170069226421, + "pass@8": 0.9696769057695721, + "pass@16": 0.9798609945844656, + "pass@32": 0.9858142874379154, + "pass@64": 0.9890143934765259, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9894642857142857, + "pass@2": 0.99220683352081, + "pass@4": 0.9938244295355938, + "pass@8": 0.9945718767910509, + "pass@16": 0.9949991404358748, + "pass@32": 0.9957142856718992, + "pass@64": 0.9971428571428571, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9928935860058309, + "pass@2": 0.9970085053143868, + "pass@4": 0.9992781772358628, + "pass@8": 0.9999411811374931, + "pass@16": 0.9999995372325835, + "pass@32": 0.9999999999908739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9961851242690059, + "pass@2": 0.9976123975457015, + "pass@4": 0.998593672227375, + "pass@8": 0.9993826865860662, + "pass@16": 0.99988448784076, + "pass@32": 0.9999973942969671, + "pass@64": 0.9999999999215047, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9512408088235295, + "pass@2": 0.9592754892311257, + "pass@4": 0.9672070549085778, + "pass@8": 0.9742943951584943, + "pass@16": 0.9797759062977376, + "pass@32": 0.9843737785787031, + "pass@64": 0.9896992479161258, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9752316640378549, + "pass@2": 0.9804542928289322, + "pass@4": 0.9846770751645001, + "pass@8": 0.9885439412426841, + "pass@16": 0.9913278776467133, + "pass@32": 0.9933394230271457, + "pass@64": 0.9950781489247041, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9796528876582279, + "pass@2": 0.985694473861258, + "pass@4": 0.9887304917581505, + "pass@8": 0.9907698483062334, + "pass@16": 0.9927418508415174, + "pass@32": 0.994748499944022, + "pass@64": 0.996357980344224, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9564329954954955, + "pass@2": 0.9685896526447708, + "pass@4": 0.9777987281432166, + "pass@8": 0.9852023893939588, + "pass@16": 0.9905368812834143, + "pass@32": 0.9943325122952712, + "pass@64": 0.9973765202133318, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.96241961414791, + "pass@2": 0.9755305782717674, + "pass@4": 0.9802783508572683, + "pass@8": 0.9834987228210043, + "pass@16": 0.9869310415394815, + "pass@32": 0.9902900110658145, + "pass@64": 0.9926323245605343, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.8774553571428572, + "pass@2": 0.9201056899137611, + "pass@4": 0.9411811779479943, + "pass@8": 0.9534647217495896, + "pass@16": 0.9630343209400213, + "pass@32": 0.9713331761600971, + "pass@64": 0.9783436058997058, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.6939506880733946, + "pass@2": 0.7832756928893064, + "pass@4": 0.8455138635193533, + "pass@8": 0.8890513361030553, + "pass@16": 0.9183177835907471, + "pass@32": 0.9375012531153086, + "pass@64": 0.9500191762654265, + "pass@128": 0.9571865443425076 + }, + "13": { + "pass@1": 0.4593460648148148, + "pass@2": 0.550352158671138, + "pass@4": 0.6256323665560319, + "pass@8": 0.6875551867759847, + "pass@16": 0.7360322879202137, + "pass@32": 0.7765612944510213, + "pass@64": 0.8141867170120144, + "pass@128": 0.8518518518518519 + }, + "14": { + "pass@1": 0.38234885014836795, + "pass@2": 0.46612764024860376, + "pass@4": 0.5396942860814502, + "pass@8": 0.6044605719250148, + "pass@16": 0.6604712523124485, + "pass@32": 0.7068676293724594, + "pass@64": 0.7443352257906309, + "pass@128": 0.7744807121661721 + }, + "15": { + "pass@1": 0.2726417824074074, + "pass@2": 0.3460583594585398, + "pass@4": 0.41217337416156297, + "pass@8": 0.4727927552375029, + "pass@16": 0.5298785998494511, + "pass@32": 0.5832519581605827, + "pass@64": 0.6289946833739761, + "pass@128": 0.6635802469135802 + }, + "16": { + "pass@1": 0.20927820796460178, + "pass@2": 0.2710314902561958, + "pass@4": 0.32689492325627434, + "pass@8": 0.37984261448337203, + "pass@16": 0.43272626830354155, + "pass@32": 0.48435329728914633, + "pass@64": 0.5332086540560117, + "pass@128": 0.5781710914454278 + }, + "17": { + "pass@1": 0.17561652861445784, + "pass@2": 0.2379966470211554, + "pass@4": 0.29631255251602584, + "pass@8": 0.3500892982483715, + "pass@16": 0.4004591753225813, + "pass@32": 0.44598628696095505, + "pass@64": 0.4889772959528953, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18268649193548386, + "pass@2": 0.24208748729997448, + "pass@4": 0.3009205241481911, + "pass@8": 0.35592454781716937, + "pass@16": 0.40402308504606516, + "pass@32": 0.44578225999860455, + "pass@64": 0.4843893762241014, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.15315804380664652, + "pass@2": 0.21352803780003343, + "pass@4": 0.27246658895734704, + "pass@8": 0.32722728747094926, + "pass@16": 0.3772414787512142, + "pass@32": 0.41963885160277065, + "pass@64": 0.4509154252350958, + "pass@128": 0.4743202416918429 + }, + "20": { + "pass@1": 0.16246715465465467, + "pass@2": 0.22468765517485983, + "pass@4": 0.284834960100708, + "pass@8": 0.33897903867127327, + "pass@16": 0.3879200126258433, + "pass@32": 0.4341533925251601, + "pass@64": 0.4796573665199342, + "pass@128": 0.5195195195195195 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..09c67c9f072157e49391acf6ae96410b3c351545 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4522fe7823b8fc35f20b3eaa41c7e084ec452939dec099c31a313f7960cf15a2 +size 2558540622 diff --git a/results/composition/SDAR-100M-block16-epoch1-op7-10/summary.csv b/results/composition/SDAR-100M-block16-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..060ad8d478659352ac6bac1d7e58f34aa744efca --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op7-10,0.9702256944444444,0.35165546875,0.6446624177631579,0.0,0.0,0.0,199.40936458333334,320.09001796875,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch1-op7-10/summary.json b/results/composition/SDAR-100M-block16-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..595e6679d2b31810b857ec968c419ea6564b4ab7 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op7-10", + "id_acc": 0.9702256944444444, + "ood_acc": 0.35165546875, + "total_acc": 0.6446624177631579, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.40936458333334, + "ood_avg_resp_len": 320.09001796875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 519453, + "answer_accuracy": 0.6415154224628518, + "avg_response_len": 250.30623246324691, + "resp_tokens_sum": 202679965, + "pass_at_k": { + "pass@1": 0.6415154224628518, + "pass@2": 0.6825554019034074, + "pass@4": 0.7162482077109957, + "pass@8": 0.7440358797725249, + "pass@16": 0.7673312676500218, + "pass@32": 0.7871519992986242, + "pass@64": 0.8040759025836397, + "pass@128": 0.8188428706923806 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9222437315634219, + "pass@2": 0.9451468248437975, + "pass@4": 0.9604848572578869, + "pass@8": 0.9712668741187709, + "pass@16": 0.9797398811861905, + "pass@32": 0.986931871980946, + "pass@64": 0.9931403532607942, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.989242601246106, + "pass@2": 0.9923153224421711, + "pass@4": 0.9941436283665476, + "pass@8": 0.9953145921083346, + "pass@16": 0.9966062996683891, + "pass@32": 0.9983101878563488, + "pass@64": 0.9997246792749038, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975630733944955, + "pass@2": 0.9996982530280046, + "pass@4": 0.9999930091880717, + "pass@8": 0.9999999967989345, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992236328125, + "pass@2": 0.9945377860482283, + "pass@4": 0.9961152392083802, + "pass@8": 0.9975110009296438, + "pass@16": 0.9987271357419916, + "pass@32": 0.9996050186206877, + "pass@64": 0.99997946273747, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9551587301587302, + "pass@2": 0.963551431071116, + "pass@4": 0.9705333979085945, + "pass@8": 0.9768221559126241, + "pass@16": 0.9826882032901565, + "pass@32": 0.988084362473456, + "pass@64": 0.992762368619646, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9563061377245509, + "pass@2": 0.9692498614974779, + "pass@4": 0.9767356600073196, + "pass@8": 0.9813628882947097, + "pass@16": 0.9850560727092349, + "pass@32": 0.9883559011032365, + "pass@64": 0.991386843357507, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9775563979289941, + "pass@2": 0.9875821908633461, + "pass@4": 0.9925620808123836, + "pass@8": 0.9950344430852842, + "pass@16": 0.9964798475945668, + "pass@32": 0.997005436160734, + "pass@64": 0.9970413979979968, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9700270432692307, + "pass@2": 0.9840835888602868, + "pass@4": 0.9896981008383567, + "pass@8": 0.9919003031750214, + "pass@16": 0.9935285960287868, + "pass@32": 0.9953138505545496, + "pass@64": 0.9966101352715523, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9554577464788733, + "pass@2": 0.9742978540534544, + "pass@4": 0.984426338080979, + "pass@8": 0.9901960111237609, + "pass@16": 0.9936839096661296, + "pass@32": 0.9957886523598386, + "pass@64": 0.9969763514894339, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8513411827195467, + "pass@2": 0.9002692054605079, + "pass@4": 0.9301656318278912, + "pass@8": 0.9489189290719003, + "pass@16": 0.9620749038932601, + "pass@32": 0.9706335966755882, + "pass@64": 0.9751169549682261, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.7018449858757062, + "pass@2": 0.786281779661017, + "pass@4": 0.8425637121101385, + "pass@8": 0.8814481070980112, + "pass@16": 0.9089087677573177, + "pass@32": 0.9300411469087887, + "pass@64": 0.9476799728882689, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.46071708463949845, + "pass@2": 0.5479957822180537, + "pass@4": 0.6168102769481397, + "pass@8": 0.6711818283871478, + "pass@16": 0.714081813122014, + "pass@32": 0.7497040752868626, + "pass@64": 0.7797252476653084, + "pass@128": 0.8056426332288401 + }, + "14": { + "pass@1": 0.3388093847352025, + "pass@2": 0.41182321178894693, + "pass@4": 0.4764041463275034, + "pass@8": 0.534130178473544, + "pass@16": 0.5871949465142853, + "pass@32": 0.6363122142046183, + "pass@64": 0.6797314428750697, + "pass@128": 0.7165109034267912 + }, + "15": { + "pass@1": 0.26275740606936415, + "pass@2": 0.3398819751035454, + "pass@4": 0.41254214727855526, + "pass@8": 0.4781416760702823, + "pass@16": 0.5331817011339766, + "pass@32": 0.5794008648263183, + "pass@64": 0.6195336061079341, + "pass@128": 0.6560693641618497 + }, + "16": { + "pass@1": 0.19909274193548387, + "pass@2": 0.26266430657861306, + "pass@4": 0.3225466130241786, + "pass@8": 0.37652892333302973, + "pass@16": 0.4252326950906888, + "pass@32": 0.46738340757399127, + "pass@64": 0.505813596566932, + "pass@128": 0.5451612903225806 + }, + "17": { + "pass@1": 0.18987084537572255, + "pass@2": 0.25301889422420454, + "pass@4": 0.30949784781237605, + "pass@8": 0.35793822819914667, + "pass@16": 0.39987978151086667, + "pass@32": 0.4364144577614477, + "pass@64": 0.46841234177173346, + "pass@128": 0.5028901734104047 + }, + "18": { + "pass@1": 0.16642857142857143, + "pass@2": 0.22563835770528673, + "pass@4": 0.2839633877015374, + "pass@8": 0.3352756128571621, + "pass@16": 0.3794013307508793, + "pass@32": 0.41912972908665974, + "pass@64": 0.45480493264252136, + "pass@128": 0.4857142857142857 + }, + "19": { + "pass@1": 0.15287162162162163, + "pass@2": 0.21274547184980255, + "pass@4": 0.2720780853969829, + "pass@8": 0.3269558100570385, + "pass@16": 0.3740083610670051, + "pass@32": 0.41088179701472477, + "pass@64": 0.44192531549939784, + "pass@128": 0.47147147147147145 + }, + "20": { + "pass@1": 0.17898367117117117, + "pass@2": 0.24102498167458797, + "pass@4": 0.2964189808481148, + "pass@8": 0.34303163036602097, + "pass@16": 0.38364439504923087, + "pass@32": 0.4206433730102397, + "pass@64": 0.4507408603585201, + "pass@128": 0.46846846846846846 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 529783, + "answer_accuracy": 0.6451955865159782, + "avg_response_len": 259.37219772992984, + "resp_tokens_sum": 212975699, + "pass_at_k": { + "pass@1": 0.6451955865159782, + "pass@2": 0.68438010153982, + "pass@4": 0.7178246167251279, + "pass@8": 0.7464661811749366, + "pass@16": 0.770699083942655, + "pass@32": 0.79120344103857, + "pass@64": 0.8090207708634686, + "pass@128": 0.8255650818394388 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9157400306748467, + "pass@2": 0.9375864239650259, + "pass@4": 0.9503189637721664, + "pass@8": 0.959476709600869, + "pass@16": 0.9678207783989702, + "pass@32": 0.976828113172386, + "pass@64": 0.9855899073216104, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9930661094224924, + "pass@2": 0.9945559497881914, + "pass@4": 0.9952527896398966, + "pass@8": 0.9960218094796784, + "pass@16": 0.996694876664963, + "pass@32": 0.9969450828883747, + "pass@64": 0.9969604795045498, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9994791666666667, + "pass@2": 0.9999657003101885, + "pass@4": 0.9999997156605425, + "pass@8": 0.9999999999936351, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9902921597633136, + "pass@2": 0.9935943542375252, + "pass@4": 0.996376267685474, + "pass@8": 0.9984966632337737, + "pass@16": 0.9996459679388677, + "pass@32": 0.9999785901153704, + "pass@64": 0.9999999845150909, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9790760869565217, + "pass@2": 0.9860735478717333, + "pass@4": 0.990809667269852, + "pass@8": 0.9943024238372346, + "pass@16": 0.9967581848072867, + "pass@32": 0.9982186985410357, + "pass@64": 0.9992791299089842, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9788905802292264, + "pass@2": 0.9836452716422623, + "pass@4": 0.9860912927610409, + "pass@8": 0.9885082645987024, + "pass@16": 0.9914576644920206, + "pass@32": 0.994320594090395, + "pass@64": 0.9964244742548366, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.979091401734104, + "pass@2": 0.9841676954849575, + "pass@4": 0.9884633377655827, + "pass@8": 0.9922804183734506, + "pass@16": 0.9951181931156187, + "pass@32": 0.9970528714118317, + "pass@64": 0.9984707380171003, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9671654929577465, + "pass@2": 0.9811702894532549, + "pass@4": 0.9894243719535057, + "pass@8": 0.9944174851855382, + "pass@16": 0.9969106947639146, + "pass@32": 0.9978444559613356, + "pass@64": 0.9985914715905386, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9666448353293413, + "pass@2": 0.9787814872224057, + "pass@4": 0.984590454074977, + "pass@8": 0.9884570828941244, + "pass@16": 0.9916722775023806, + "pass@32": 0.9940966746741055, + "pass@64": 0.9954845654245074, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.867211031626506, + "pass@2": 0.911044522578503, + "pass@4": 0.9335228710116054, + "pass@8": 0.9456478076783712, + "pass@16": 0.9524671555395998, + "pass@32": 0.9570610201613988, + "pass@64": 0.9621282660404785, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.71608052507837, + "pass@2": 0.7989217967072296, + "pass@4": 0.8551101025185328, + "pass@8": 0.8918423946053244, + "pass@16": 0.9147090576639251, + "pass@32": 0.9288851299053897, + "pass@64": 0.9409932385294371, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.40434611344537813, + "pass@2": 0.48740185050398094, + "pass@4": 0.5640730444408734, + "pass@8": 0.6355232978685984, + "pass@16": 0.6996316769009948, + "pass@32": 0.7527484934731532, + "pass@64": 0.7943710656136685, + "pass@128": 0.8291316526610645 + }, + "14": { + "pass@1": 0.3604486476608187, + "pass@2": 0.44008114322880676, + "pass@4": 0.5081644520312152, + "pass@8": 0.5657154899960264, + "pass@16": 0.6164087304791761, + "pass@32": 0.6613545821335254, + "pass@64": 0.6976279435761811, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.22407670454545456, + "pass@2": 0.30122323132903833, + "pass@4": 0.37661032541386885, + "pass@8": 0.44410643751587464, + "pass@16": 0.4993273457645693, + "pass@32": 0.5427207028934755, + "pass@64": 0.5788425426868631, + "pass@128": 0.6121212121212121 + }, + "16": { + "pass@1": 0.23039084757834757, + "pass@2": 0.2978291074993831, + "pass@4": 0.35624147569160675, + "pass@8": 0.40731401792660654, + "pass@16": 0.45254311760056604, + "pass@32": 0.49147289445246195, + "pass@64": 0.5235553530937181, + "pass@128": 0.5470085470085471 + }, + "17": { + "pass@1": 0.21370341614906832, + "pass@2": 0.2806520363623024, + "pass@4": 0.3440671235971277, + "pass@8": 0.40228302586205095, + "pass@16": 0.4537379226594085, + "pass@32": 0.500026224710816, + "pass@64": 0.5454292278608023, + "pass@128": 0.593167701863354 + }, + "18": { + "pass@1": 0.17212775735294117, + "pass@2": 0.23196720125057887, + "pass@4": 0.2933669291338581, + "pass@8": 0.35235556676313684, + "pass@16": 0.4034011348916748, + "pass@32": 0.4468507861160033, + "pass@64": 0.48950582673115856, + "pass@128": 0.5352941176470588 + }, + "19": { + "pass@1": 0.15087890625, + "pass@2": 0.2072612017247844, + "pass@4": 0.2605428718731588, + "pass@8": 0.3074317250275432, + "pass@16": 0.34908958061974404, + "pass@32": 0.3879378881426714, + "pass@64": 0.42367879660082997, + "pass@128": 0.45535714285714285 + }, + "20": { + "pass@1": 0.1435254491017964, + "pass@2": 0.20275848401621951, + "pass@4": 0.26188721965517786, + "pass@8": 0.31475370332114117, + "pass@16": 0.3611758484951675, + "pass@32": 0.40307471926923255, + "pass@64": 0.43896682078335747, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 518583, + "answer_accuracy": 0.6472966428343185, + "avg_response_len": 279.3217104868989, + "resp_tokens_sum": 223779147, + "pass_at_k": { + "pass@1": 0.6472966428343185, + "pass@2": 0.687566046625146, + "pass@4": 0.7213202171624994, + "pass@8": 0.7503762891676571, + "pass@16": 0.7756265763304547, + "pass@32": 0.7975734180227835, + "pass@64": 0.8168150755671897, + "pass@128": 0.8335197315865154 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9071595149253732, + "pass@2": 0.9365723057938655, + "pass@4": 0.9556170069226421, + "pass@8": 0.9696769057695721, + "pass@16": 0.9798609945844656, + "pass@32": 0.9858142874379154, + "pass@64": 0.9890143934765259, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9894642857142857, + "pass@2": 0.99220683352081, + "pass@4": 0.9938244295355938, + "pass@8": 0.9945718767910509, + "pass@16": 0.9949991404358748, + "pass@32": 0.9957142856718992, + "pass@64": 0.9971428571428571, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9928935860058309, + "pass@2": 0.9970085053143868, + "pass@4": 0.9992781772358628, + "pass@8": 0.9999411811374931, + "pass@16": 0.9999995372325835, + "pass@32": 0.9999999999908739, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9961851242690059, + "pass@2": 0.9976123975457015, + "pass@4": 0.998593672227375, + "pass@8": 0.9993826865860662, + "pass@16": 0.99988448784076, + "pass@32": 0.9999973942969671, + "pass@64": 0.9999999999215047, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9512408088235295, + "pass@2": 0.9592754892311257, + "pass@4": 0.9672070549085778, + "pass@8": 0.9742943951584943, + "pass@16": 0.9797759062977376, + "pass@32": 0.9843737785787031, + "pass@64": 0.9896992479161258, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9752316640378549, + "pass@2": 0.9804542928289322, + "pass@4": 0.9846770751645001, + "pass@8": 0.9885439412426841, + "pass@16": 0.9913278776467133, + "pass@32": 0.9933394230271457, + "pass@64": 0.9950781489247041, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9796528876582279, + "pass@2": 0.985694473861258, + "pass@4": 0.9887304917581505, + "pass@8": 0.9907698483062334, + "pass@16": 0.9927418508415174, + "pass@32": 0.994748499944022, + "pass@64": 0.996357980344224, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9564329954954955, + "pass@2": 0.9685896526447708, + "pass@4": 0.9777987281432166, + "pass@8": 0.9852023893939588, + "pass@16": 0.9905368812834143, + "pass@32": 0.9943325122952712, + "pass@64": 0.9973765202133318, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.96241961414791, + "pass@2": 0.9755305782717674, + "pass@4": 0.9802783508572683, + "pass@8": 0.9834987228210043, + "pass@16": 0.9869310415394815, + "pass@32": 0.9902900110658145, + "pass@64": 0.9926323245605343, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.8774553571428572, + "pass@2": 0.9201056899137611, + "pass@4": 0.9411811779479943, + "pass@8": 0.9534647217495896, + "pass@16": 0.9630343209400213, + "pass@32": 0.9713331761600971, + "pass@64": 0.9783436058997058, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.6939506880733946, + "pass@2": 0.7832756928893064, + "pass@4": 0.8455138635193533, + "pass@8": 0.8890513361030553, + "pass@16": 0.9183177835907471, + "pass@32": 0.9375012531153086, + "pass@64": 0.9500191762654265, + "pass@128": 0.9571865443425076 + }, + "13": { + "pass@1": 0.4593460648148148, + "pass@2": 0.550352158671138, + "pass@4": 0.6256323665560319, + "pass@8": 0.6875551867759847, + "pass@16": 0.7360322879202137, + "pass@32": 0.7765612944510213, + "pass@64": 0.8141867170120144, + "pass@128": 0.8518518518518519 + }, + "14": { + "pass@1": 0.38234885014836795, + "pass@2": 0.46612764024860376, + "pass@4": 0.5396942860814502, + "pass@8": 0.6044605719250148, + "pass@16": 0.6604712523124485, + "pass@32": 0.7068676293724594, + "pass@64": 0.7443352257906309, + "pass@128": 0.7744807121661721 + }, + "15": { + "pass@1": 0.2726417824074074, + "pass@2": 0.3460583594585398, + "pass@4": 0.41217337416156297, + "pass@8": 0.4727927552375029, + "pass@16": 0.5298785998494511, + "pass@32": 0.5832519581605827, + "pass@64": 0.6289946833739761, + "pass@128": 0.6635802469135802 + }, + "16": { + "pass@1": 0.20927820796460178, + "pass@2": 0.2710314902561958, + "pass@4": 0.32689492325627434, + "pass@8": 0.37984261448337203, + "pass@16": 0.43272626830354155, + "pass@32": 0.48435329728914633, + "pass@64": 0.5332086540560117, + "pass@128": 0.5781710914454278 + }, + "17": { + "pass@1": 0.17561652861445784, + "pass@2": 0.2379966470211554, + "pass@4": 0.29631255251602584, + "pass@8": 0.3500892982483715, + "pass@16": 0.4004591753225813, + "pass@32": 0.44598628696095505, + "pass@64": 0.4889772959528953, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18268649193548386, + "pass@2": 0.24208748729997448, + "pass@4": 0.3009205241481911, + "pass@8": 0.35592454781716937, + "pass@16": 0.40402308504606516, + "pass@32": 0.44578225999860455, + "pass@64": 0.4843893762241014, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.15315804380664652, + "pass@2": 0.21352803780003343, + "pass@4": 0.27246658895734704, + "pass@8": 0.32722728747094926, + "pass@16": 0.3772414787512142, + "pass@32": 0.41963885160277065, + "pass@64": 0.4509154252350958, + "pass@128": 0.4743202416918429 + }, + "20": { + "pass@1": 0.16246715465465467, + "pass@2": 0.22468765517485983, + "pass@4": 0.284834960100708, + "pass@8": 0.33897903867127327, + "pass@16": 0.3879200126258433, + "pass@32": 0.4341533925251601, + "pass@64": 0.4796573665199342, + "pass@128": 0.5195195195195195 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..eab792198aa2f6bea258f8f247b2e39220d4f517 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f437557eb8eecb54fa5efde88957ea338a563d8852db4b057dff138f33de7be2 +size 1792857151 diff --git a/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json b/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..633ca10a333cbc7778d29840e648606228944f1d --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9653489583333333, + "avg_response_len": 199.63633854166667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9653489583333333, + "pass@2": 0.9760651110017445, + "pass@4": 0.9827205345269335, + "pass@8": 0.9872042269695149, + "pass@16": 0.990437182192509, + "pass@32": 0.9929618421362073, + "pass@64": 0.994864366664554, + "pass@128": 0.996 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9176640625, + "pass@2": 0.940686146653544, + "pass@4": 0.9572751454818151, + "pass@8": 0.9697146074131044, + "pass@16": 0.9786371901621834, + "pass@32": 0.9846897936578861, + "pass@64": 0.9886130167733074, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.988515625, + "pass@2": 0.9917391732283464, + "pass@4": 0.9931179612860892, + "pass@8": 0.9939276029971762, + "pass@16": 0.9945250737957444, + "pass@32": 0.995013888448554, + "pass@64": 0.9954711788593589, + "pass@128": 0.996 + }, + "3": { + "pass@1": 0.993640625, + "pass@2": 0.997156619094488, + "pass@4": 0.9985739075740535, + "pass@8": 0.9990783257375846, + "pass@16": 0.9993271435678454, + "pass@32": 0.9995814496671865, + "pass@64": 0.9998779527559054, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9908671875, + "pass@2": 0.9935773868110238, + "pass@4": 0.9958510903637046, + "pass@8": 0.9974042594396715, + "pass@16": 0.9983575185071549, + "pass@32": 0.9991406729482755, + "pass@64": 0.9997315728158566, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9576875, + "pass@2": 0.9662279773622043, + "pass@4": 0.9724250765841771, + "pass@8": 0.977084930700057, + "pass@16": 0.981167244820512, + "pass@32": 0.9850588622178171, + "pass@64": 0.9886003349090352, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.9621640625, + "pass@2": 0.9733558070866145, + "pass@4": 0.9805783572365954, + "pass@8": 0.9853265539365899, + "pass@16": 0.9881276610501337, + "pass@32": 0.9902692290508687, + "pass@64": 0.9921569678703794, + "pass@128": 0.993 + }, + "7": { + "pass@1": 0.969, + "pass@2": 0.9790697588582676, + "pass@4": 0.9853942632170978, + "pass@8": 0.989212366296118, + "pass@16": 0.9919174897126309, + "pass@32": 0.9943850588471859, + "pass@64": 0.9962791607703998, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9544375, + "pass@2": 0.9708692175196855, + "pass@4": 0.9808557603112115, + "pass@8": 0.9876466681787328, + "pass@16": 0.992855800260977, + "pass@32": 0.9967386933633018, + "pass@64": 0.9989564749136509, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9541640625, + "pass@2": 0.9719039124015749, + "pass@4": 0.9804132486876638, + "pass@8": 0.9854427280266015, + "pass@16": 0.9890195178553929, + "pass@32": 0.99177893102479, + "pass@64": 0.9940926403130936, + "pass@128": 0.996 + } + }, + "per_op_accuracy": { + "10": 0.9176640625, + "2": 0.988515625, + "3": 0.993640625, + "4": 0.9908671875, + "5": 0.9576875, + "6": 0.9621640625, + "7": 0.969, + "8": 0.9544375, + "9": 0.9541640625 + }, + "per_op_avg_response_len": { + "10": 278.3240859375, + "2": 116.5158515625, + "3": 144.4954140625, + "4": 159.5400078125, + "5": 174.3931875, + "6": 199.3752578125, + "7": 223.4614296875, + "8": 242.72865625, + "9": 257.89315625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.3240859375, + "2": 116.5158515625, + "3": 144.4954140625, + "4": 159.5400078125, + "5": 174.3931875, + "6": 199.3752578125, + "7": 223.4614296875, + "8": 242.72865625, + "9": 257.89315625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 364498, + "answer_accuracy": 0.9617158476865924, + "avg_response_len": 190.16717852921312, + "resp_tokens_sum": 72074882, + "pass_at_k": { + "pass@1": 0.9617158476865924, + "pass@2": 0.9740216149842975, + "pass@4": 0.9816819834894001, + "pass@8": 0.9866704765932208, + "pass@16": 0.9901795736650141, + "pass@32": 0.9931229258326321, + "pass@64": 0.9954760430923487, + "pass@128": 0.9966227625802094 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9215984513274337, + "pass@2": 0.9421719305274897, + "pass@4": 0.9578695823973329, + "pass@8": 0.9698981562700201, + "pass@16": 0.9786144975781491, + "pass@32": 0.9852228034323779, + "pass@64": 0.9899066079157088, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9882204049844237, + "pass@2": 0.9918561581671449, + "pass@4": 0.9931930275654794, + "pass@8": 0.9936832086849384, + "pass@16": 0.993767744760047, + "pass@32": 0.9937694702012028, + "pass@64": 0.9937694704049844, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9939554663608563, + "pass@2": 0.9985781772977917, + "pass@4": 0.999890842199771, + "pass@8": 0.9999992999127619, + "pass@16": 0.9999999999827538, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990234375, + "pass@2": 0.99342742679626, + "pass@4": 0.9958457609790964, + "pass@8": 0.9970974256312388, + "pass@16": 0.9976077641280157, + "pass@32": 0.9982468010091103, + "pass@64": 0.9992249015748031, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.948139880952381, + "pass@2": 0.9574174321959756, + "pass@4": 0.9648101639080825, + "pass@8": 0.971654004642766, + "pass@16": 0.9780823665716158, + "pass@32": 0.9842143450259111, + "pass@64": 0.9898102076387585, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9406811377245509, + "pass@2": 0.9580407344759303, + "pass@4": 0.9713076608313183, + "pass@8": 0.9802488166001808, + "pass@16": 0.9856429524545877, + "pass@32": 0.9897026831061002, + "pass@64": 0.9927742684651953, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.96875, + "pass@2": 0.980806722615664, + "pass@4": 0.988206052553342, + "pass@8": 0.9921328277288592, + "pass@16": 0.9942489886848938, + "pass@32": 0.9958450340249061, + "pass@64": 0.9968560505325343, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.955879407051282, + "pass@2": 0.9741089680496671, + "pass@4": 0.9835598903502444, + "pass@8": 0.9884050733900349, + "pass@16": 0.9918600204545246, + "pass@32": 0.9954140492926473, + "pass@64": 0.9986018037303472, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9509242957746479, + "pass@2": 0.9714289675058222, + "pass@4": 0.9813418648373179, + "pass@8": 0.9873102931793774, + "pass@16": 0.991888158345391, + "pass@32": 0.9956463578845761, + "pass@64": 0.9982995101172762, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 378550, + "answer_accuracy": 0.9690110992791612, + "avg_response_len": 197.52059868528835, + "resp_tokens_sum": 77162607, + "pass_at_k": { + "pass@1": 0.9690110992791612, + "pass@2": 0.978732119315075, + "pass@4": 0.9846058772060452, + "pass@8": 0.9885791851961677, + "pass@16": 0.991478696028919, + "pass@32": 0.9936181979373836, + "pass@64": 0.9950741600217173, + "pass@128": 0.9960681520314548 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9162432898773006, + "pass@2": 0.936796154171296, + "pass@4": 0.9512743250843644, + "pass@8": 0.9633956921405639, + "pass@16": 0.9736279940224598, + "pass@32": 0.9804423382093927, + "pass@64": 0.9842310694963466, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9920687689969605, + "pass@2": 0.9949422462484743, + "pass@4": 0.9961704575112003, + "pass@8": 0.9969137768076869, + "pass@16": 0.997325481498739, + "pass@32": 0.9977203377335374, + "pass@64": 0.9984802431610939, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942708333333333, + "pass@2": 0.9958694971367216, + "pass@4": 0.9967973148242834, + "pass@8": 0.9974456186860212, + "pass@16": 0.9979759402219975, + "pass@32": 0.9987316925844025, + "pass@64": 0.9996301598663804, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875416050295858, + "pass@2": 0.9915100987746355, + "pass@4": 0.9949397878519625, + "pass@8": 0.9972912179822564, + "pass@16": 0.9987250305872443, + "pass@32": 0.9996231767711692, + "pass@64": 0.9999805562449308, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9740942028985508, + "pass@2": 0.9819546673513638, + "pass@4": 0.987201654956174, + "pass@8": 0.9904344185485051, + "pass@16": 0.9930448124857775, + "pass@32": 0.9952773429073588, + "pass@64": 0.9967067319810339, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9741001074498568, + "pass@2": 0.982676178282156, + "pass@4": 0.9865603822265768, + "pass@8": 0.9885311263764657, + "pass@16": 0.9894516215507114, + "pass@32": 0.9902045618641498, + "pass@64": 0.9910543058907616, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9690435332369942, + "pass@2": 0.9785164250603066, + "pass@4": 0.9848848110929772, + "pass@8": 0.9886472236910547, + "pass@16": 0.991308776141372, + "pass@32": 0.993989628100775, + "pass@64": 0.9961800005089887, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9597271126760564, + "pass@2": 0.9766656038593767, + "pass@4": 0.9864835997612974, + "pass@8": 0.992742027879815, + "pass@16": 0.9970229470709905, + "pass@32": 0.9994615576509764, + "pass@64": 0.9999920569141826, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9533589071856288, + "pass@2": 0.9688030435192612, + "pass@4": 0.9762362204724409, + "pass@8": 0.9810118633763555, + "pass@16": 0.9841736264077786, + "pass@32": 0.9865662118269425, + "pass@64": 0.988977219442525, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 369034, + "answer_accuracy": 0.9652086123200536, + "avg_response_len": 211.1848557289923, + "resp_tokens_sum": 80743573, + "pass_at_k": { + "pass@1": 0.9652086123200536, + "pass@2": 0.9753657746560565, + "pass@4": 0.981823676062086, + "pass@8": 0.986328452733312, + "pass@16": 0.9896283702143321, + "pass@32": 0.9921315218381473, + "pass@64": 0.9940436558413996, + "pass@128": 0.9953130231001004 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9150652985074627, + "pass@2": 0.9429681072981553, + "pass@4": 0.9625132151391523, + "pass@8": 0.9756780202977416, + "pass@16": 0.9835347744234902, + "pass@32": 0.9882837642926804, + "pass@64": 0.9915682030867851, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9854464285714286, + "pass@2": 0.988620992688414, + "pass@4": 0.9901797683325299, + "pass@8": 0.991344744113091, + "pass@16": 0.9925872694705262, + "pass@32": 0.9936111354275262, + "pass@64": 0.9942033681695974, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9927341472303207, + "pass@2": 0.9970397115539129, + "pass@4": 0.9990276626318211, + "pass@8": 0.9997711384831617, + "pass@16": 0.9999856655983261, + "pass@32": 0.9999999740942677, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9947459795321637, + "pass@2": 0.9957608095040752, + "pass@4": 0.9967567209252353, + "pass@8": 0.997803074735884, + "pass@16": 0.9986958293792437, + "pass@32": 0.9995001838500145, + "pass@64": 0.9999595798278741, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9498851102941176, + "pass@2": 0.9584327234830942, + "pass@4": 0.9644861882154439, + "pass@8": 0.968570720112799, + "pass@16": 0.9719730560084113, + "pass@32": 0.9754725300929893, + "pass@64": 0.979253814615793, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9716581230283912, + "pass@2": 0.979230963883852, + "pass@4": 0.9837603316146996, + "pass@8": 0.9871485365512419, + "pass@16": 0.9892880094924388, + "pass@32": 0.9909373526903568, + "pass@64": 0.992720534533528, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9692197389240507, + "pass@2": 0.9778177327319845, + "pass@4": 0.9829445342275255, + "pass@8": 0.9867073769830967, + "pass@16": 0.9900901740260195, + "pass@32": 0.9932564114680369, + "pass@64": 0.995770682007257, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9474474474474475, + "pass@2": 0.96165444479913, + "pass@4": 0.9723225123323547, + "pass@8": 0.9815041002513741, + "pass@16": 0.9893463293362276, + "pass@32": 0.9950770481017992, + "pass@64": 0.9981847806163591, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9587268890675241, + "pass@2": 0.9757762456642275, + "pass@4": 0.983839193030935, + "pass@8": 0.9880694906116392, + "pass@16": 0.9909492939632827, + "pass@32": 0.9929625698571264, + "pass@64": 0.9947843180953608, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4310265625, + "avg_response_len": 339.695190625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4310265625, + "pass@2": 0.5022684793307036, + "pass@4": 0.5631944049868662, + "pass@8": 0.6146080622451209, + "pass@16": 0.658063651886774, + "pass@32": 0.6956868152191318, + "pass@64": 0.7288434322106975, + "pass@128": 0.7572 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.897328125, + "pass@2": 0.93245250984252, + "pass@4": 0.9525756509186349, + "pass@8": 0.9645405289253838, + "pass@16": 0.9721646148438103, + "pass@32": 0.9774677539333159, + "pass@64": 0.9818126562253906, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.83025, + "pass@2": 0.8788818897637795, + "pass@4": 0.9077505954255725, + "pass@8": 0.9279219926000699, + "pass@16": 0.9429205001103443, + "pass@32": 0.9540556965113227, + "pass@64": 0.9624023478061977, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.6266796875, + "pass@2": 0.7173414124015755, + "pass@4": 0.7835768498312717, + "pass@8": 0.8310638378335047, + "pass@16": 0.8652153962130704, + "pass@32": 0.8921147478710862, + "pass@64": 0.9136172681639906, + "pass@128": 0.93 + }, + "14": { + "pass@1": 0.5015, + "pass@2": 0.589535925196851, + "pass@4": 0.6631350288713906, + "pass@8": 0.7234786940771628, + "pass@16": 0.7724374351005907, + "pass@32": 0.8117817021972529, + "pass@64": 0.8433248048181223, + "pass@128": 0.869 + }, + "15": { + "pass@1": 0.385046875, + "pass@2": 0.47474187992126016, + "pass@4": 0.5506543599550056, + "pass@8": 0.6135148960829805, + "pass@16": 0.6656588575667085, + "pass@32": 0.7095550865297331, + "pass@64": 0.7464043576049708, + "pass@128": 0.775 + }, + "16": { + "pass@1": 0.280765625, + "pass@2": 0.3638841043307086, + "pass@4": 0.43851758830146187, + "pass@8": 0.5031006368868065, + "pass@16": 0.5591009613517215, + "pass@32": 0.6095957808194802, + "pass@64": 0.6556664450191428, + "pass@128": 0.694 + }, + "17": { + "pass@1": 0.2475625, + "pass@2": 0.3250761564960633, + "pass@4": 0.39567444216347986, + "pass@8": 0.45756658041957493, + "pass@16": 0.5128672951589743, + "pass@32": 0.5644771191304162, + "pass@64": 0.6129670466840049, + "pass@128": 0.656 + }, + "18": { + "pass@1": 0.21215625, + "pass@2": 0.2847347440944884, + "pass@4": 0.35507883173978216, + "pass@8": 0.4180793864445228, + "pass@16": 0.4726955368933959, + "pass@32": 0.5197814062778024, + "pass@64": 0.5618646594106671, + "pass@128": 0.601 + }, + "19": { + "pass@1": 0.1663125, + "pass@2": 0.22953531003937006, + "pass@4": 0.2925731046119234, + "pass@8": 0.35215195152385936, + "pass@16": 0.4065216822086127, + "pass@32": 0.4569250687655832, + "pass@64": 0.5050905644534417, + "pass@128": 0.55 + }, + "20": { + "pass@1": 0.1626640625, + "pass@2": 0.22650086122047253, + "pass@4": 0.2924075980502435, + "pass@8": 0.3546621176573502, + "pass@16": 0.41105423942037367, + "pass@32": 0.4611137901553134, + "pass@64": 0.505284171920995, + "pass@128": 0.543 + } + }, + "per_op_accuracy": { + "11": 0.897328125, + "12": 0.83025, + "13": 0.6266796875, + "14": 0.5015, + "15": 0.385046875, + "16": 0.280765625, + "17": 0.2475625, + "18": 0.21215625, + "19": 0.1663125, + "20": 0.1626640625 + }, + "per_op_avg_response_len": { + "11": 295.7106484375, + "12": 306.1600703125, + "13": 321.2292265625, + "14": 337.66740625, + "15": 344.118015625, + "16": 350.866875, + "17": 360.0460703125, + "18": 359.24096875, + "19": 359.20896875, + "20": 362.70365625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 295.7106484375, + "12": 306.1600703125, + "13": 321.2292265625, + "14": 337.66740625, + "15": 344.118015625, + "16": 350.866875, + "17": 360.0460703125, + "18": 359.24096875, + "19": 359.20896875, + "20": 362.70365625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 186658, + "answer_accuracy": 0.43336274145616643, + "avg_response_len": 322.15230544205053, + "resp_tokens_sum": 138757441, + "pass_at_k": { + "pass@1": 0.43336274145616643, + "pass@2": 0.5036887145932549, + "pass@4": 0.563023118395492, + "pass@8": 0.6126057036749238, + "pass@16": 0.6542006347449212, + "pass@32": 0.6899950752564268, + "pass@64": 0.7213229495910691, + "pass@128": 0.7465081723625557 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8825469192634561, + "pass@2": 0.9204746018380142, + "pass@4": 0.9434641893205277, + "pass@8": 0.9584269403349234, + "pass@16": 0.9686552276126651, + "pass@32": 0.9741207162725538, + "pass@64": 0.9764454185753144, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.8271539548022598, + "pass@2": 0.8798437569509316, + "pass@4": 0.9131463289016838, + "pass@8": 0.9351818057985669, + "pass@16": 0.9499145912403741, + "pass@32": 0.9615198784184202, + "pass@64": 0.9709481160243175, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.6312695924764891, + "pass@2": 0.7226765945498974, + "pass@4": 0.7903836963561368, + "pass@8": 0.838930594168827, + "pass@16": 0.8732006163685243, + "pass@32": 0.9012810590070837, + "pass@64": 0.9258364518367349, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.4956678348909657, + "pass@2": 0.5795753152059264, + "pass@4": 0.6470877296879942, + "pass@8": 0.7036891591817919, + "pass@16": 0.7513465261882352, + "pass@32": 0.7915273102114493, + "pass@64": 0.8268042808049906, + "pass@128": 0.8598130841121495 + }, + "15": { + "pass@1": 0.3917765534682081, + "pass@2": 0.47504665240544336, + "pass@4": 0.5456971947328837, + "pass@8": 0.6072410800646598, + "pass@16": 0.6594419230378626, + "pass@32": 0.7028996498231647, + "pass@64": 0.7405890497226499, + "pass@128": 0.7658959537572254 + }, + "16": { + "pass@1": 0.26927923387096775, + "pass@2": 0.3519208788417577, + "pass@4": 0.42269466175599024, + "pass@8": 0.4817823935686336, + "pass@16": 0.5343682207500322, + "pass@32": 0.582402508764003, + "pass@64": 0.6250428507581135, + "pass@128": 0.6612903225806451 + }, + "17": { + "pass@1": 0.25090317919075145, + "pass@2": 0.3269800248054254, + "pass@4": 0.39415536226324305, + "pass@8": 0.45047162566799187, + "pass@16": 0.4981191025185966, + "pass@32": 0.5403608962010116, + "pass@64": 0.5799445864824998, + "pass@128": 0.6184971098265896 + }, + "18": { + "pass@1": 0.21515625, + "pass@2": 0.28855982845894257, + "pass@4": 0.3581241681396968, + "pass@8": 0.4172596510922582, + "pass@16": 0.46627537268369557, + "pass@32": 0.5088608026804442, + "pass@64": 0.5478183836857043, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.16713588588588588, + "pass@2": 0.23041065179352582, + "pass@4": 0.2955489223756941, + "pass@8": 0.35859808935671944, + "pass@16": 0.4160070631029604, + "pass@32": 0.468307236198052, + "pass@64": 0.5149739853037154, + "pass@128": 0.5495495495495496 + }, + "20": { + "pass@1": 0.17004504504504506, + "pass@2": 0.23310330507673024, + "pass@4": 0.2956246063273622, + "pass@8": 0.35363622885925766, + "pass@16": 0.4070566006447026, + "pass@32": 0.4541781885859068, + "pass@64": 0.4931356276630924, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 183610, + "answer_accuracy": 0.42653973386856975, + "avg_response_len": 334.73815696550696, + "resp_tokens_sum": 144092726, + "pass_at_k": { + "pass@1": 0.42653973386856975, + "pass@2": 0.4969607598671056, + "pass@4": 0.557970116246622, + "pass@8": 0.6104187474444682, + "pass@16": 0.6550365457245686, + "pass@32": 0.6931812007460348, + "pass@64": 0.7262481004101062, + "pass@128": 0.7555753791257805 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9048616340361446, + "pass@2": 0.9367618110236223, + "pass@4": 0.9549433162910352, + "pass@8": 0.964925632387157, + "pass@16": 0.9708902210043483, + "pass@32": 0.9769209821868238, + "pass@64": 0.9840171200594384, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8286637931034483, + "pass@2": 0.8699518518747071, + "pass@4": 0.8934778219337001, + "pass@8": 0.9107875074125524, + "pass@16": 0.924846496240714, + "pass@32": 0.936201270194174, + "pass@64": 0.9454492653916154, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6203825280112045, + "pass@2": 0.7105237764397101, + "pass@4": 0.7767815107670366, + "pass@8": 0.8251601495483123, + "pass@16": 0.860732014990156, + "pass@32": 0.8879965216471163, + "pass@64": 0.9088056396870849, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.4910453216374269, + "pass@2": 0.5799452905557857, + "pass@4": 0.6568933461168229, + "pass@8": 0.7221252947493525, + "pass@16": 0.7770240520072819, + "pass@32": 0.8219059508512782, + "pass@64": 0.8560598866085424, + "pass@128": 0.8830409356725146 + }, + "15": { + "pass@1": 0.3689157196969697, + "pass@2": 0.46035552374135, + "pass@4": 0.5369870879776393, + "pass@8": 0.5998998198723953, + "pass@16": 0.6521363093899889, + "pass@32": 0.6950081588345737, + "pass@64": 0.7272367323897406, + "pass@128": 0.7515151515151515 + }, + "16": { + "pass@1": 0.278957443019943, + "pass@2": 0.3574475065616797, + "pass@4": 0.4265418227636073, + "pass@8": 0.48653231264461394, + "pass@16": 0.5386797703066138, + "pass@32": 0.585372520779792, + "pass@64": 0.6303605102283197, + "pass@128": 0.6723646723646723 + }, + "17": { + "pass@1": 0.2574970885093168, + "pass@2": 0.33939995781777277, + "pass@4": 0.4177574955692649, + "pass@8": 0.49007486766441977, + "pass@16": 0.5536131251915021, + "pass@32": 0.6097010001622978, + "pass@64": 0.6588642656789311, + "pass@128": 0.6987577639751553 + }, + "18": { + "pass@1": 0.21059283088235295, + "pass@2": 0.2836838524779991, + "pass@4": 0.3568823333664176, + "pass@8": 0.4261368301347266, + "pass@16": 0.48815849016843194, + "pass@32": 0.5409295253300673, + "pass@64": 0.5854362229138341, + "pass@128": 0.6235294117647059 + }, + "19": { + "pass@1": 0.16171409970238096, + "pass@2": 0.2210187593738281, + "pass@4": 0.2790568570111771, + "pass@8": 0.3330461307196623, + "pass@16": 0.3823068390231134, + "pass@32": 0.4286068727382764, + "pass@64": 0.47582930473787466, + "pass@128": 0.5297619047619048 + }, + "20": { + "pass@1": 0.15501029191616766, + "pass@2": 0.21969520604460377, + "pass@4": 0.28707018359232045, + "pass@8": 0.35047540956249357, + "pass@16": 0.40577476078592467, + "pass@32": 0.4521128631680825, + "pass@64": 0.492629793596181, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 181446, + "answer_accuracy": 0.4332355974938875, + "avg_response_len": 362.83159430394255, + "resp_tokens_sum": 151959677, + "pass_at_k": { + "pass@1": 0.4332355974938875, + "pass@2": 0.5062632131013821, + "pass@4": 0.5687401453333036, + "pass@8": 0.6209731607363563, + "pass@16": 0.6651477627381525, + "pass@32": 0.7041156313705635, + "pass@64": 0.7392451818623957, + "pass@128": 0.7698655256723717 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.905952380952381, + "pass@2": 0.9413335051868518, + "pass@4": 0.960290828824968, + "pass@8": 0.9709857429670468, + "pass@16": 0.9774405273749097, + "pass@32": 0.9817948412796146, + "pass@64": 0.9855039352653046, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8351490825688074, + "pass@2": 0.8865521683642764, + "pass@4": 0.9158329351606272, + "pass@8": 0.93677803817362, + "pass@16": 0.9529807416222166, + "pass@32": 0.9633928267867302, + "pass@64": 0.969689293803253, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6290991512345679, + "pass@2": 0.7196005881209291, + "pass@4": 0.7843625041661457, + "pass@8": 0.8298234719287071, + "pass@16": 0.8622934266667467, + "pass@32": 0.887627567345078, + "pass@64": 0.9068883538882164, + "pass@128": 0.9228395061728395 + }, + "14": { + "pass@1": 0.517665059347181, + "pass@2": 0.6087565568120749, + "pass@4": 0.684754727802941, + "pass@8": 0.7437021459214515, + "pass@16": 0.787872327619222, + "pass@32": 0.820800001270744, + "pass@64": 0.8461369420166132, + "pass@128": 0.8635014836795252 + }, + "15": { + "pass@1": 0.3942901234567901, + "pass@2": 0.4890691831923788, + "pass@4": 0.5698684924222436, + "pass@8": 0.6340819192059179, + "pass@16": 0.6860708953608408, + "pass@32": 0.731478750850336, + "pass@64": 0.7721371133096265, + "pass@128": 0.808641975308642 + }, + "16": { + "pass@1": 0.29314159292035397, + "pass@2": 0.3814883689870623, + "pass@4": 0.46538661760200317, + "pass@8": 0.5397500092692332, + "pass@16": 0.6028619868483489, + "pass@32": 0.6595435053360812, + "pass@64": 0.7098720418701685, + "pass@128": 0.7463126843657817 + }, + "17": { + "pass@1": 0.23444559487951808, + "pass@2": 0.30919964306043063, + "pass@4": 0.37583967845510263, + "pass@8": 0.4334315980436936, + "pass@16": 0.48871885354179795, + "pass@32": 0.5457485752789358, + "pass@64": 0.6028672476278413, + "pass@128": 0.6536144578313253 + }, + "18": { + "pass@1": 0.21048387096774193, + "pass@2": 0.2815686912623825, + "pass@4": 0.34966251466550535, + "pass@8": 0.41016769779492335, + "pass@16": 0.46298474128011696, + "pass@32": 0.5089164087981424, + "pass@64": 0.5518706752579567, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.17015200151057402, + "pass@2": 0.23729987867830718, + "pass@4": 0.3032997266013954, + "pass@8": 0.36506129258690445, + "pass@16": 0.42155961414972953, + "pass@32": 0.4742200902464383, + "pass@64": 0.5248506977503865, + "pass@128": 0.5709969788519638 + }, + "20": { + "pass@1": 0.16295983483483484, + "pass@2": 0.22672450994301377, + "pass@4": 0.2945440325026939, + "pass@8": 0.35988728724728264, + "pass@16": 0.4203472111206872, + "pass@32": 0.4770773485227238, + "pass@64": 0.5301250956398212, + "pass@128": 0.5765765765765766 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6841266447368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6841266447368421, + "pass@2": 0.7266984627538122, + "pass@4": 0.7619173084531972, + "pass@8": 0.7911009823777397, + "pass@16": 0.8155037451894985, + "pass@32": 0.8365013016535509, + "pass@64": 0.8548533485309445, + "pass@128": 0.8703157894736843 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9176640625, + "pass@2": 0.940686146653544, + "pass@4": 0.9572751454818151, + "pass@8": 0.9697146074131044, + "pass@16": 0.9786371901621834, + "pass@32": 0.9846897936578861, + "pass@64": 0.9886130167733074, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.988515625, + "pass@2": 0.9917391732283464, + "pass@4": 0.9931179612860892, + "pass@8": 0.9939276029971762, + "pass@16": 0.9945250737957444, + "pass@32": 0.995013888448554, + "pass@64": 0.9954711788593589, + "pass@128": 0.996 + }, + "3": { + "pass@1": 0.993640625, + "pass@2": 0.997156619094488, + "pass@4": 0.9985739075740535, + "pass@8": 0.9990783257375846, + "pass@16": 0.9993271435678454, + "pass@32": 0.9995814496671865, + "pass@64": 0.9998779527559054, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9908671875, + "pass@2": 0.9935773868110238, + "pass@4": 0.9958510903637046, + "pass@8": 0.9974042594396715, + "pass@16": 0.9983575185071549, + "pass@32": 0.9991406729482755, + "pass@64": 0.9997315728158566, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9576875, + "pass@2": 0.9662279773622043, + "pass@4": 0.9724250765841771, + "pass@8": 0.977084930700057, + "pass@16": 0.981167244820512, + "pass@32": 0.9850588622178171, + "pass@64": 0.9886003349090352, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.9621640625, + "pass@2": 0.9733558070866145, + "pass@4": 0.9805783572365954, + "pass@8": 0.9853265539365899, + "pass@16": 0.9881276610501337, + "pass@32": 0.9902692290508687, + "pass@64": 0.9921569678703794, + "pass@128": 0.993 + }, + "7": { + "pass@1": 0.969, + "pass@2": 0.9790697588582676, + "pass@4": 0.9853942632170978, + "pass@8": 0.989212366296118, + "pass@16": 0.9919174897126309, + "pass@32": 0.9943850588471859, + "pass@64": 0.9962791607703998, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9544375, + "pass@2": 0.9708692175196855, + "pass@4": 0.9808557603112115, + "pass@8": 0.9876466681787328, + "pass@16": 0.992855800260977, + "pass@32": 0.9967386933633018, + "pass@64": 0.9989564749136509, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9541640625, + "pass@2": 0.9719039124015749, + "pass@4": 0.9804132486876638, + "pass@8": 0.9854427280266015, + "pass@16": 0.9890195178553929, + "pass@32": 0.99177893102479, + "pass@64": 0.9940926403130936, + "pass@128": 0.996 + }, + "11": { + "pass@1": 0.897328125, + "pass@2": 0.93245250984252, + "pass@4": 0.9525756509186349, + "pass@8": 0.9645405289253838, + "pass@16": 0.9721646148438103, + "pass@32": 0.9774677539333159, + "pass@64": 0.9818126562253906, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.83025, + "pass@2": 0.8788818897637795, + "pass@4": 0.9077505954255725, + "pass@8": 0.9279219926000699, + "pass@16": 0.9429205001103443, + "pass@32": 0.9540556965113227, + "pass@64": 0.9624023478061977, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.6266796875, + "pass@2": 0.7173414124015755, + "pass@4": 0.7835768498312717, + "pass@8": 0.8310638378335047, + "pass@16": 0.8652153962130704, + "pass@32": 0.8921147478710862, + "pass@64": 0.9136172681639906, + "pass@128": 0.93 + }, + "14": { + "pass@1": 0.5015, + "pass@2": 0.589535925196851, + "pass@4": 0.6631350288713906, + "pass@8": 0.7234786940771628, + "pass@16": 0.7724374351005907, + "pass@32": 0.8117817021972529, + "pass@64": 0.8433248048181223, + "pass@128": 0.869 + }, + "15": { + "pass@1": 0.385046875, + "pass@2": 0.47474187992126016, + "pass@4": 0.5506543599550056, + "pass@8": 0.6135148960829805, + "pass@16": 0.6656588575667085, + "pass@32": 0.7095550865297331, + "pass@64": 0.7464043576049708, + "pass@128": 0.775 + }, + "16": { + "pass@1": 0.280765625, + "pass@2": 0.3638841043307086, + "pass@4": 0.43851758830146187, + "pass@8": 0.5031006368868065, + "pass@16": 0.5591009613517215, + "pass@32": 0.6095957808194802, + "pass@64": 0.6556664450191428, + "pass@128": 0.694 + }, + "17": { + "pass@1": 0.2475625, + "pass@2": 0.3250761564960633, + "pass@4": 0.39567444216347986, + "pass@8": 0.45756658041957493, + "pass@16": 0.5128672951589743, + "pass@32": 0.5644771191304162, + "pass@64": 0.6129670466840049, + "pass@128": 0.656 + }, + "18": { + "pass@1": 0.21215625, + "pass@2": 0.2847347440944884, + "pass@4": 0.35507883173978216, + "pass@8": 0.4180793864445228, + "pass@16": 0.4726955368933959, + "pass@32": 0.5197814062778024, + "pass@64": 0.5618646594106671, + "pass@128": 0.601 + }, + "19": { + "pass@1": 0.1663125, + "pass@2": 0.22953531003937006, + "pass@4": 0.2925731046119234, + "pass@8": 0.35215195152385936, + "pass@16": 0.4065216822086127, + "pass@32": 0.4569250687655832, + "pass@64": 0.5050905644534417, + "pass@128": 0.55 + }, + "20": { + "pass@1": 0.1626640625, + "pass@2": 0.22650086122047253, + "pass@4": 0.2924075980502435, + "pass@8": 0.3546621176573502, + "pass@16": 0.41105423942037367, + "pass@32": 0.4611137901553134, + "pass@64": 0.505284171920995, + "pass@128": 0.543 + } + }, + "per_op_accuracy": { + "10": 0.9176640625, + "2": 0.988515625, + "3": 0.993640625, + "4": 0.9908671875, + "5": 0.9576875, + "6": 0.9621640625, + "7": 0.969, + "8": 0.9544375, + "9": 0.9541640625, + "11": 0.897328125, + "12": 0.83025, + "13": 0.6266796875, + "14": 0.5015, + "15": 0.385046875, + "16": 0.280765625, + "17": 0.2475625, + "18": 0.21215625, + "19": 0.1663125, + "20": 0.1626640625 + }, + "per_op_avg_response_len": { + "10": 278.3240859375, + "2": 116.5158515625, + "3": 144.4954140625, + "4": 159.5400078125, + "5": 174.3931875, + "6": 199.3752578125, + "7": 223.4614296875, + "8": 242.72865625, + "9": 257.89315625, + "11": 295.7106484375, + "12": 306.1600703125, + "13": 321.2292265625, + "14": 337.66740625, + "15": 344.118015625, + "16": 350.866875, + "17": 360.0460703125, + "18": 359.24096875, + "19": 359.20896875, + "20": 362.70365625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.3240859375, + "2": 116.5158515625, + "3": 144.4954140625, + "4": 159.5400078125, + "5": 174.3931875, + "6": 199.3752578125, + "7": 223.4614296875, + "8": 242.72865625, + "9": 257.89315625, + "11": 295.7106484375, + "12": 306.1600703125, + "13": 321.2292265625, + "14": 337.66740625, + "15": 344.118015625, + "16": 350.866875, + "17": 360.0460703125, + "18": 359.24096875, + "19": 359.20896875, + "20": 362.70365625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 551156, + "answer_accuracy": 0.6806680761934872, + "avg_response_len": 260.37425283552005, + "resp_tokens_sum": 210832323, + "pass_at_k": { + "pass@1": 0.6806680761934872, + "pass@2": 0.7238366308211794, + "pass@4": 0.7589840573052321, + "pass@8": 0.7876935621338345, + "pass@16": 0.8114617220263678, + "pass@32": 0.8318796098052982, + "pass@64": 0.8496453191543497, + "pass@128": 0.8635788808093582 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9215984513274337, + "pass@2": 0.9421719305274897, + "pass@4": 0.9578695823973329, + "pass@8": 0.9698981562700201, + "pass@16": 0.9786144975781491, + "pass@32": 0.9852228034323779, + "pass@64": 0.9899066079157088, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9882204049844237, + "pass@2": 0.9918561581671449, + "pass@4": 0.9931930275654794, + "pass@8": 0.9936832086849384, + "pass@16": 0.993767744760047, + "pass@32": 0.9937694702012028, + "pass@64": 0.9937694704049844, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9939554663608563, + "pass@2": 0.9985781772977917, + "pass@4": 0.999890842199771, + "pass@8": 0.9999992999127619, + "pass@16": 0.9999999999827538, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990234375, + "pass@2": 0.99342742679626, + "pass@4": 0.9958457609790964, + "pass@8": 0.9970974256312388, + "pass@16": 0.9976077641280157, + "pass@32": 0.9982468010091103, + "pass@64": 0.9992249015748031, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.948139880952381, + "pass@2": 0.9574174321959756, + "pass@4": 0.9648101639080825, + "pass@8": 0.971654004642766, + "pass@16": 0.9780823665716158, + "pass@32": 0.9842143450259111, + "pass@64": 0.9898102076387585, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9406811377245509, + "pass@2": 0.9580407344759303, + "pass@4": 0.9713076608313183, + "pass@8": 0.9802488166001808, + "pass@16": 0.9856429524545877, + "pass@32": 0.9897026831061002, + "pass@64": 0.9927742684651953, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.96875, + "pass@2": 0.980806722615664, + "pass@4": 0.988206052553342, + "pass@8": 0.9921328277288592, + "pass@16": 0.9942489886848938, + "pass@32": 0.9958450340249061, + "pass@64": 0.9968560505325343, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.955879407051282, + "pass@2": 0.9741089680496671, + "pass@4": 0.9835598903502444, + "pass@8": 0.9884050733900349, + "pass@16": 0.9918600204545246, + "pass@32": 0.9954140492926473, + "pass@64": 0.9986018037303472, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9509242957746479, + "pass@2": 0.9714289675058222, + "pass@4": 0.9813418648373179, + "pass@8": 0.9873102931793774, + "pass@16": 0.991888158345391, + "pass@32": 0.9956463578845761, + "pass@64": 0.9982995101172762, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8825469192634561, + "pass@2": 0.9204746018380142, + "pass@4": 0.9434641893205277, + "pass@8": 0.9584269403349234, + "pass@16": 0.9686552276126651, + "pass@32": 0.9741207162725538, + "pass@64": 0.9764454185753144, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.8271539548022598, + "pass@2": 0.8798437569509316, + "pass@4": 0.9131463289016838, + "pass@8": 0.9351818057985669, + "pass@16": 0.9499145912403741, + "pass@32": 0.9615198784184202, + "pass@64": 0.9709481160243175, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.6312695924764891, + "pass@2": 0.7226765945498974, + "pass@4": 0.7903836963561368, + "pass@8": 0.838930594168827, + "pass@16": 0.8732006163685243, + "pass@32": 0.9012810590070837, + "pass@64": 0.9258364518367349, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.4956678348909657, + "pass@2": 0.5795753152059264, + "pass@4": 0.6470877296879942, + "pass@8": 0.7036891591817919, + "pass@16": 0.7513465261882352, + "pass@32": 0.7915273102114493, + "pass@64": 0.8268042808049906, + "pass@128": 0.8598130841121495 + }, + "15": { + "pass@1": 0.3917765534682081, + "pass@2": 0.47504665240544336, + "pass@4": 0.5456971947328837, + "pass@8": 0.6072410800646598, + "pass@16": 0.6594419230378626, + "pass@32": 0.7028996498231647, + "pass@64": 0.7405890497226499, + "pass@128": 0.7658959537572254 + }, + "16": { + "pass@1": 0.26927923387096775, + "pass@2": 0.3519208788417577, + "pass@4": 0.42269466175599024, + "pass@8": 0.4817823935686336, + "pass@16": 0.5343682207500322, + "pass@32": 0.582402508764003, + "pass@64": 0.6250428507581135, + "pass@128": 0.6612903225806451 + }, + "17": { + "pass@1": 0.25090317919075145, + "pass@2": 0.3269800248054254, + "pass@4": 0.39415536226324305, + "pass@8": 0.45047162566799187, + "pass@16": 0.4981191025185966, + "pass@32": 0.5403608962010116, + "pass@64": 0.5799445864824998, + "pass@128": 0.6184971098265896 + }, + "18": { + "pass@1": 0.21515625, + "pass@2": 0.28855982845894257, + "pass@4": 0.3581241681396968, + "pass@8": 0.4172596510922582, + "pass@16": 0.46627537268369557, + "pass@32": 0.5088608026804442, + "pass@64": 0.5478183836857043, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.16713588588588588, + "pass@2": 0.23041065179352582, + "pass@4": 0.2955489223756941, + "pass@8": 0.35859808935671944, + "pass@16": 0.4160070631029604, + "pass@32": 0.468307236198052, + "pass@64": 0.5149739853037154, + "pass@128": 0.5495495495495496 + }, + "20": { + "pass@1": 0.17004504504504506, + "pass@2": 0.23310330507673024, + "pass@4": 0.2956246063273622, + "pass@8": 0.35363622885925766, + "pass@16": 0.4070566006447026, + "pass@32": 0.4541781885859068, + "pass@64": 0.4931356276630924, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 562160, + "answer_accuracy": 0.6846258768511302, + "avg_response_len": 269.4555399941543, + "resp_tokens_sum": 221255333, + "pass_at_k": { + "pass@1": 0.6846258768511302, + "pass@2": 0.7261682717977604, + "pass@4": 0.7609463192782817, + "pass@8": 0.7903323337294538, + "pass@16": 0.8151022421749073, + "pass@32": 0.8361170877963872, + "pass@64": 0.8541447697685913, + "pass@128": 0.8699922057677318 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9162432898773006, + "pass@2": 0.936796154171296, + "pass@4": 0.9512743250843644, + "pass@8": 0.9633956921405639, + "pass@16": 0.9736279940224598, + "pass@32": 0.9804423382093927, + "pass@64": 0.9842310694963466, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9920687689969605, + "pass@2": 0.9949422462484743, + "pass@4": 0.9961704575112003, + "pass@8": 0.9969137768076869, + "pass@16": 0.997325481498739, + "pass@32": 0.9977203377335374, + "pass@64": 0.9984802431610939, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942708333333333, + "pass@2": 0.9958694971367216, + "pass@4": 0.9967973148242834, + "pass@8": 0.9974456186860212, + "pass@16": 0.9979759402219975, + "pass@32": 0.9987316925844025, + "pass@64": 0.9996301598663804, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875416050295858, + "pass@2": 0.9915100987746355, + "pass@4": 0.9949397878519625, + "pass@8": 0.9972912179822564, + "pass@16": 0.9987250305872443, + "pass@32": 0.9996231767711692, + "pass@64": 0.9999805562449308, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9740942028985508, + "pass@2": 0.9819546673513638, + "pass@4": 0.987201654956174, + "pass@8": 0.9904344185485051, + "pass@16": 0.9930448124857775, + "pass@32": 0.9952773429073588, + "pass@64": 0.9967067319810339, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9741001074498568, + "pass@2": 0.982676178282156, + "pass@4": 0.9865603822265768, + "pass@8": 0.9885311263764657, + "pass@16": 0.9894516215507114, + "pass@32": 0.9902045618641498, + "pass@64": 0.9910543058907616, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9690435332369942, + "pass@2": 0.9785164250603066, + "pass@4": 0.9848848110929772, + "pass@8": 0.9886472236910547, + "pass@16": 0.991308776141372, + "pass@32": 0.993989628100775, + "pass@64": 0.9961800005089887, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9597271126760564, + "pass@2": 0.9766656038593767, + "pass@4": 0.9864835997612974, + "pass@8": 0.992742027879815, + "pass@16": 0.9970229470709905, + "pass@32": 0.9994615576509764, + "pass@64": 0.9999920569141826, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9533589071856288, + "pass@2": 0.9688030435192612, + "pass@4": 0.9762362204724409, + "pass@8": 0.9810118633763555, + "pass@16": 0.9841736264077786, + "pass@32": 0.9865662118269425, + "pass@64": 0.988977219442525, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9048616340361446, + "pass@2": 0.9367618110236223, + "pass@4": 0.9549433162910352, + "pass@8": 0.964925632387157, + "pass@16": 0.9708902210043483, + "pass@32": 0.9769209821868238, + "pass@64": 0.9840171200594384, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8286637931034483, + "pass@2": 0.8699518518747071, + "pass@4": 0.8934778219337001, + "pass@8": 0.9107875074125524, + "pass@16": 0.924846496240714, + "pass@32": 0.936201270194174, + "pass@64": 0.9454492653916154, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6203825280112045, + "pass@2": 0.7105237764397101, + "pass@4": 0.7767815107670366, + "pass@8": 0.8251601495483123, + "pass@16": 0.860732014990156, + "pass@32": 0.8879965216471163, + "pass@64": 0.9088056396870849, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.4910453216374269, + "pass@2": 0.5799452905557857, + "pass@4": 0.6568933461168229, + "pass@8": 0.7221252947493525, + "pass@16": 0.7770240520072819, + "pass@32": 0.8219059508512782, + "pass@64": 0.8560598866085424, + "pass@128": 0.8830409356725146 + }, + "15": { + "pass@1": 0.3689157196969697, + "pass@2": 0.46035552374135, + "pass@4": 0.5369870879776393, + "pass@8": 0.5998998198723953, + "pass@16": 0.6521363093899889, + "pass@32": 0.6950081588345737, + "pass@64": 0.7272367323897406, + "pass@128": 0.7515151515151515 + }, + "16": { + "pass@1": 0.278957443019943, + "pass@2": 0.3574475065616797, + "pass@4": 0.4265418227636073, + "pass@8": 0.48653231264461394, + "pass@16": 0.5386797703066138, + "pass@32": 0.585372520779792, + "pass@64": 0.6303605102283197, + "pass@128": 0.6723646723646723 + }, + "17": { + "pass@1": 0.2574970885093168, + "pass@2": 0.33939995781777277, + "pass@4": 0.4177574955692649, + "pass@8": 0.49007486766441977, + "pass@16": 0.5536131251915021, + "pass@32": 0.6097010001622978, + "pass@64": 0.6588642656789311, + "pass@128": 0.6987577639751553 + }, + "18": { + "pass@1": 0.21059283088235295, + "pass@2": 0.2836838524779991, + "pass@4": 0.3568823333664176, + "pass@8": 0.4261368301347266, + "pass@16": 0.48815849016843194, + "pass@32": 0.5409295253300673, + "pass@64": 0.5854362229138341, + "pass@128": 0.6235294117647059 + }, + "19": { + "pass@1": 0.16171409970238096, + "pass@2": 0.2210187593738281, + "pass@4": 0.2790568570111771, + "pass@8": 0.3330461307196623, + "pass@16": 0.3823068390231134, + "pass@32": 0.4286068727382764, + "pass@64": 0.47582930473787466, + "pass@128": 0.5297619047619048 + }, + "20": { + "pass@1": 0.15501029191616766, + "pass@2": 0.21969520604460377, + "pass@4": 0.28707018359232045, + "pass@8": 0.35047540956249357, + "pass@16": 0.40577476078592467, + "pass@32": 0.4521128631680825, + "pass@64": 0.492629793596181, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 550480, + "answer_accuracy": 0.6871105607924589, + "avg_response_len": 290.4607989495127, + "resp_tokens_sum": 232703250, + "pass_at_k": { + "pass@1": 0.6871105607924589, + "pass@2": 0.7301343349041897, + "pass@4": 0.7658771490538376, + "pass@8": 0.7953326841737876, + "pass@16": 0.8200005466543361, + "pass@32": 0.8415662568421538, + "pass@64": 0.8608433671596183, + "pass@128": 0.877456462693721 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9150652985074627, + "pass@2": 0.9429681072981553, + "pass@4": 0.9625132151391523, + "pass@8": 0.9756780202977416, + "pass@16": 0.9835347744234902, + "pass@32": 0.9882837642926804, + "pass@64": 0.9915682030867851, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9854464285714286, + "pass@2": 0.988620992688414, + "pass@4": 0.9901797683325299, + "pass@8": 0.991344744113091, + "pass@16": 0.9925872694705262, + "pass@32": 0.9936111354275262, + "pass@64": 0.9942033681695974, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9927341472303207, + "pass@2": 0.9970397115539129, + "pass@4": 0.9990276626318211, + "pass@8": 0.9997711384831617, + "pass@16": 0.9999856655983261, + "pass@32": 0.9999999740942677, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9947459795321637, + "pass@2": 0.9957608095040752, + "pass@4": 0.9967567209252353, + "pass@8": 0.997803074735884, + "pass@16": 0.9986958293792437, + "pass@32": 0.9995001838500145, + "pass@64": 0.9999595798278741, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9498851102941176, + "pass@2": 0.9584327234830942, + "pass@4": 0.9644861882154439, + "pass@8": 0.968570720112799, + "pass@16": 0.9719730560084113, + "pass@32": 0.9754725300929893, + "pass@64": 0.979253814615793, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9716581230283912, + "pass@2": 0.979230963883852, + "pass@4": 0.9837603316146996, + "pass@8": 0.9871485365512419, + "pass@16": 0.9892880094924388, + "pass@32": 0.9909373526903568, + "pass@64": 0.992720534533528, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9692197389240507, + "pass@2": 0.9778177327319845, + "pass@4": 0.9829445342275255, + "pass@8": 0.9867073769830967, + "pass@16": 0.9900901740260195, + "pass@32": 0.9932564114680369, + "pass@64": 0.995770682007257, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9474474474474475, + "pass@2": 0.96165444479913, + "pass@4": 0.9723225123323547, + "pass@8": 0.9815041002513741, + "pass@16": 0.9893463293362276, + "pass@32": 0.9950770481017992, + "pass@64": 0.9981847806163591, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9587268890675241, + "pass@2": 0.9757762456642275, + "pass@4": 0.983839193030935, + "pass@8": 0.9880694906116392, + "pass@16": 0.9909492939632827, + "pass@32": 0.9929625698571264, + "pass@64": 0.9947843180953608, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.905952380952381, + "pass@2": 0.9413335051868518, + "pass@4": 0.960290828824968, + "pass@8": 0.9709857429670468, + "pass@16": 0.9774405273749097, + "pass@32": 0.9817948412796146, + "pass@64": 0.9855039352653046, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8351490825688074, + "pass@2": 0.8865521683642764, + "pass@4": 0.9158329351606272, + "pass@8": 0.93677803817362, + "pass@16": 0.9529807416222166, + "pass@32": 0.9633928267867302, + "pass@64": 0.969689293803253, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6290991512345679, + "pass@2": 0.7196005881209291, + "pass@4": 0.7843625041661457, + "pass@8": 0.8298234719287071, + "pass@16": 0.8622934266667467, + "pass@32": 0.887627567345078, + "pass@64": 0.9068883538882164, + "pass@128": 0.9228395061728395 + }, + "14": { + "pass@1": 0.517665059347181, + "pass@2": 0.6087565568120749, + "pass@4": 0.684754727802941, + "pass@8": 0.7437021459214515, + "pass@16": 0.787872327619222, + "pass@32": 0.820800001270744, + "pass@64": 0.8461369420166132, + "pass@128": 0.8635014836795252 + }, + "15": { + "pass@1": 0.3942901234567901, + "pass@2": 0.4890691831923788, + "pass@4": 0.5698684924222436, + "pass@8": 0.6340819192059179, + "pass@16": 0.6860708953608408, + "pass@32": 0.731478750850336, + "pass@64": 0.7721371133096265, + "pass@128": 0.808641975308642 + }, + "16": { + "pass@1": 0.29314159292035397, + "pass@2": 0.3814883689870623, + "pass@4": 0.46538661760200317, + "pass@8": 0.5397500092692332, + "pass@16": 0.6028619868483489, + "pass@32": 0.6595435053360812, + "pass@64": 0.7098720418701685, + "pass@128": 0.7463126843657817 + }, + "17": { + "pass@1": 0.23444559487951808, + "pass@2": 0.30919964306043063, + "pass@4": 0.37583967845510263, + "pass@8": 0.4334315980436936, + "pass@16": 0.48871885354179795, + "pass@32": 0.5457485752789358, + "pass@64": 0.6028672476278413, + "pass@128": 0.6536144578313253 + }, + "18": { + "pass@1": 0.21048387096774193, + "pass@2": 0.2815686912623825, + "pass@4": 0.34966251466550535, + "pass@8": 0.41016769779492335, + "pass@16": 0.46298474128011696, + "pass@32": 0.5089164087981424, + "pass@64": 0.5518706752579567, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.17015200151057402, + "pass@2": 0.23729987867830718, + "pass@4": 0.3032997266013954, + "pass@8": 0.36506129258690445, + "pass@16": 0.42155961414972953, + "pass@32": 0.4742200902464383, + "pass@64": 0.5248506977503865, + "pass@128": 0.5709969788519638 + }, + "20": { + "pass@1": 0.16295983483483484, + "pass@2": 0.22672450994301377, + "pass@4": 0.2945440325026939, + "pass@8": 0.35988728724728264, + "pass@16": 0.4203472111206872, + "pass@32": 0.4770773485227238, + "pass@64": 0.5301250956398212, + "pass@128": 0.5765765765765766 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1c471ed03eee8c232bb0969e32d86a54f8dc9d91 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab0b360194a5945c6ff2a02d4c9ae86e2b4d71f892d8d5f2be441320d97008a +size 2616201721 diff --git a/results/composition/SDAR-100M-block16-epoch1-op9-12/summary.csv b/results/composition/SDAR-100M-block16-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d548ecc011b1e09415c5e2ac4a52b26b60d24607 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op9-12,0.9653489583333333,0.4310265625,0.6841266447368421,0.0,0.0,0.0,199.63633854166667,339.695190625,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch1-op9-12/summary.json b/results/composition/SDAR-100M-block16-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..951ed2f3b4bb776af8261848ccea93d136c1c9f8 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op9-12", + "id_acc": 0.9653489583333333, + "ood_acc": 0.4310265625, + "total_acc": 0.6841266447368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.63633854166667, + "ood_avg_resp_len": 339.695190625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 551156, + "answer_accuracy": 0.6806680761934872, + "avg_response_len": 260.37425283552005, + "resp_tokens_sum": 210832323, + "pass_at_k": { + "pass@1": 0.6806680761934872, + "pass@2": 0.7238366308211794, + "pass@4": 0.7589840573052321, + "pass@8": 0.7876935621338345, + "pass@16": 0.8114617220263678, + "pass@32": 0.8318796098052982, + "pass@64": 0.8496453191543497, + "pass@128": 0.8635788808093582 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9215984513274337, + "pass@2": 0.9421719305274897, + "pass@4": 0.9578695823973329, + "pass@8": 0.9698981562700201, + "pass@16": 0.9786144975781491, + "pass@32": 0.9852228034323779, + "pass@64": 0.9899066079157088, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9882204049844237, + "pass@2": 0.9918561581671449, + "pass@4": 0.9931930275654794, + "pass@8": 0.9936832086849384, + "pass@16": 0.993767744760047, + "pass@32": 0.9937694702012028, + "pass@64": 0.9937694704049844, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9939554663608563, + "pass@2": 0.9985781772977917, + "pass@4": 0.999890842199771, + "pass@8": 0.9999992999127619, + "pass@16": 0.9999999999827538, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990234375, + "pass@2": 0.99342742679626, + "pass@4": 0.9958457609790964, + "pass@8": 0.9970974256312388, + "pass@16": 0.9976077641280157, + "pass@32": 0.9982468010091103, + "pass@64": 0.9992249015748031, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.948139880952381, + "pass@2": 0.9574174321959756, + "pass@4": 0.9648101639080825, + "pass@8": 0.971654004642766, + "pass@16": 0.9780823665716158, + "pass@32": 0.9842143450259111, + "pass@64": 0.9898102076387585, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9406811377245509, + "pass@2": 0.9580407344759303, + "pass@4": 0.9713076608313183, + "pass@8": 0.9802488166001808, + "pass@16": 0.9856429524545877, + "pass@32": 0.9897026831061002, + "pass@64": 0.9927742684651953, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.96875, + "pass@2": 0.980806722615664, + "pass@4": 0.988206052553342, + "pass@8": 0.9921328277288592, + "pass@16": 0.9942489886848938, + "pass@32": 0.9958450340249061, + "pass@64": 0.9968560505325343, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.955879407051282, + "pass@2": 0.9741089680496671, + "pass@4": 0.9835598903502444, + "pass@8": 0.9884050733900349, + "pass@16": 0.9918600204545246, + "pass@32": 0.9954140492926473, + "pass@64": 0.9986018037303472, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9509242957746479, + "pass@2": 0.9714289675058222, + "pass@4": 0.9813418648373179, + "pass@8": 0.9873102931793774, + "pass@16": 0.991888158345391, + "pass@32": 0.9956463578845761, + "pass@64": 0.9982995101172762, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8825469192634561, + "pass@2": 0.9204746018380142, + "pass@4": 0.9434641893205277, + "pass@8": 0.9584269403349234, + "pass@16": 0.9686552276126651, + "pass@32": 0.9741207162725538, + "pass@64": 0.9764454185753144, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.8271539548022598, + "pass@2": 0.8798437569509316, + "pass@4": 0.9131463289016838, + "pass@8": 0.9351818057985669, + "pass@16": 0.9499145912403741, + "pass@32": 0.9615198784184202, + "pass@64": 0.9709481160243175, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.6312695924764891, + "pass@2": 0.7226765945498974, + "pass@4": 0.7903836963561368, + "pass@8": 0.838930594168827, + "pass@16": 0.8732006163685243, + "pass@32": 0.9012810590070837, + "pass@64": 0.9258364518367349, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.4956678348909657, + "pass@2": 0.5795753152059264, + "pass@4": 0.6470877296879942, + "pass@8": 0.7036891591817919, + "pass@16": 0.7513465261882352, + "pass@32": 0.7915273102114493, + "pass@64": 0.8268042808049906, + "pass@128": 0.8598130841121495 + }, + "15": { + "pass@1": 0.3917765534682081, + "pass@2": 0.47504665240544336, + "pass@4": 0.5456971947328837, + "pass@8": 0.6072410800646598, + "pass@16": 0.6594419230378626, + "pass@32": 0.7028996498231647, + "pass@64": 0.7405890497226499, + "pass@128": 0.7658959537572254 + }, + "16": { + "pass@1": 0.26927923387096775, + "pass@2": 0.3519208788417577, + "pass@4": 0.42269466175599024, + "pass@8": 0.4817823935686336, + "pass@16": 0.5343682207500322, + "pass@32": 0.582402508764003, + "pass@64": 0.6250428507581135, + "pass@128": 0.6612903225806451 + }, + "17": { + "pass@1": 0.25090317919075145, + "pass@2": 0.3269800248054254, + "pass@4": 0.39415536226324305, + "pass@8": 0.45047162566799187, + "pass@16": 0.4981191025185966, + "pass@32": 0.5403608962010116, + "pass@64": 0.5799445864824998, + "pass@128": 0.6184971098265896 + }, + "18": { + "pass@1": 0.21515625, + "pass@2": 0.28855982845894257, + "pass@4": 0.3581241681396968, + "pass@8": 0.4172596510922582, + "pass@16": 0.46627537268369557, + "pass@32": 0.5088608026804442, + "pass@64": 0.5478183836857043, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.16713588588588588, + "pass@2": 0.23041065179352582, + "pass@4": 0.2955489223756941, + "pass@8": 0.35859808935671944, + "pass@16": 0.4160070631029604, + "pass@32": 0.468307236198052, + "pass@64": 0.5149739853037154, + "pass@128": 0.5495495495495496 + }, + "20": { + "pass@1": 0.17004504504504506, + "pass@2": 0.23310330507673024, + "pass@4": 0.2956246063273622, + "pass@8": 0.35363622885925766, + "pass@16": 0.4070566006447026, + "pass@32": 0.4541781885859068, + "pass@64": 0.4931356276630924, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 562160, + "answer_accuracy": 0.6846258768511302, + "avg_response_len": 269.4555399941543, + "resp_tokens_sum": 221255333, + "pass_at_k": { + "pass@1": 0.6846258768511302, + "pass@2": 0.7261682717977604, + "pass@4": 0.7609463192782817, + "pass@8": 0.7903323337294538, + "pass@16": 0.8151022421749073, + "pass@32": 0.8361170877963872, + "pass@64": 0.8541447697685913, + "pass@128": 0.8699922057677318 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9162432898773006, + "pass@2": 0.936796154171296, + "pass@4": 0.9512743250843644, + "pass@8": 0.9633956921405639, + "pass@16": 0.9736279940224598, + "pass@32": 0.9804423382093927, + "pass@64": 0.9842310694963466, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9920687689969605, + "pass@2": 0.9949422462484743, + "pass@4": 0.9961704575112003, + "pass@8": 0.9969137768076869, + "pass@16": 0.997325481498739, + "pass@32": 0.9977203377335374, + "pass@64": 0.9984802431610939, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942708333333333, + "pass@2": 0.9958694971367216, + "pass@4": 0.9967973148242834, + "pass@8": 0.9974456186860212, + "pass@16": 0.9979759402219975, + "pass@32": 0.9987316925844025, + "pass@64": 0.9996301598663804, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875416050295858, + "pass@2": 0.9915100987746355, + "pass@4": 0.9949397878519625, + "pass@8": 0.9972912179822564, + "pass@16": 0.9987250305872443, + "pass@32": 0.9996231767711692, + "pass@64": 0.9999805562449308, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9740942028985508, + "pass@2": 0.9819546673513638, + "pass@4": 0.987201654956174, + "pass@8": 0.9904344185485051, + "pass@16": 0.9930448124857775, + "pass@32": 0.9952773429073588, + "pass@64": 0.9967067319810339, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9741001074498568, + "pass@2": 0.982676178282156, + "pass@4": 0.9865603822265768, + "pass@8": 0.9885311263764657, + "pass@16": 0.9894516215507114, + "pass@32": 0.9902045618641498, + "pass@64": 0.9910543058907616, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9690435332369942, + "pass@2": 0.9785164250603066, + "pass@4": 0.9848848110929772, + "pass@8": 0.9886472236910547, + "pass@16": 0.991308776141372, + "pass@32": 0.993989628100775, + "pass@64": 0.9961800005089887, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9597271126760564, + "pass@2": 0.9766656038593767, + "pass@4": 0.9864835997612974, + "pass@8": 0.992742027879815, + "pass@16": 0.9970229470709905, + "pass@32": 0.9994615576509764, + "pass@64": 0.9999920569141826, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9533589071856288, + "pass@2": 0.9688030435192612, + "pass@4": 0.9762362204724409, + "pass@8": 0.9810118633763555, + "pass@16": 0.9841736264077786, + "pass@32": 0.9865662118269425, + "pass@64": 0.988977219442525, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9048616340361446, + "pass@2": 0.9367618110236223, + "pass@4": 0.9549433162910352, + "pass@8": 0.964925632387157, + "pass@16": 0.9708902210043483, + "pass@32": 0.9769209821868238, + "pass@64": 0.9840171200594384, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8286637931034483, + "pass@2": 0.8699518518747071, + "pass@4": 0.8934778219337001, + "pass@8": 0.9107875074125524, + "pass@16": 0.924846496240714, + "pass@32": 0.936201270194174, + "pass@64": 0.9454492653916154, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6203825280112045, + "pass@2": 0.7105237764397101, + "pass@4": 0.7767815107670366, + "pass@8": 0.8251601495483123, + "pass@16": 0.860732014990156, + "pass@32": 0.8879965216471163, + "pass@64": 0.9088056396870849, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.4910453216374269, + "pass@2": 0.5799452905557857, + "pass@4": 0.6568933461168229, + "pass@8": 0.7221252947493525, + "pass@16": 0.7770240520072819, + "pass@32": 0.8219059508512782, + "pass@64": 0.8560598866085424, + "pass@128": 0.8830409356725146 + }, + "15": { + "pass@1": 0.3689157196969697, + "pass@2": 0.46035552374135, + "pass@4": 0.5369870879776393, + "pass@8": 0.5998998198723953, + "pass@16": 0.6521363093899889, + "pass@32": 0.6950081588345737, + "pass@64": 0.7272367323897406, + "pass@128": 0.7515151515151515 + }, + "16": { + "pass@1": 0.278957443019943, + "pass@2": 0.3574475065616797, + "pass@4": 0.4265418227636073, + "pass@8": 0.48653231264461394, + "pass@16": 0.5386797703066138, + "pass@32": 0.585372520779792, + "pass@64": 0.6303605102283197, + "pass@128": 0.6723646723646723 + }, + "17": { + "pass@1": 0.2574970885093168, + "pass@2": 0.33939995781777277, + "pass@4": 0.4177574955692649, + "pass@8": 0.49007486766441977, + "pass@16": 0.5536131251915021, + "pass@32": 0.6097010001622978, + "pass@64": 0.6588642656789311, + "pass@128": 0.6987577639751553 + }, + "18": { + "pass@1": 0.21059283088235295, + "pass@2": 0.2836838524779991, + "pass@4": 0.3568823333664176, + "pass@8": 0.4261368301347266, + "pass@16": 0.48815849016843194, + "pass@32": 0.5409295253300673, + "pass@64": 0.5854362229138341, + "pass@128": 0.6235294117647059 + }, + "19": { + "pass@1": 0.16171409970238096, + "pass@2": 0.2210187593738281, + "pass@4": 0.2790568570111771, + "pass@8": 0.3330461307196623, + "pass@16": 0.3823068390231134, + "pass@32": 0.4286068727382764, + "pass@64": 0.47582930473787466, + "pass@128": 0.5297619047619048 + }, + "20": { + "pass@1": 0.15501029191616766, + "pass@2": 0.21969520604460377, + "pass@4": 0.28707018359232045, + "pass@8": 0.35047540956249357, + "pass@16": 0.40577476078592467, + "pass@32": 0.4521128631680825, + "pass@64": 0.492629793596181, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 550480, + "answer_accuracy": 0.6871105607924589, + "avg_response_len": 290.4607989495127, + "resp_tokens_sum": 232703250, + "pass_at_k": { + "pass@1": 0.6871105607924589, + "pass@2": 0.7301343349041897, + "pass@4": 0.7658771490538376, + "pass@8": 0.7953326841737876, + "pass@16": 0.8200005466543361, + "pass@32": 0.8415662568421538, + "pass@64": 0.8608433671596183, + "pass@128": 0.877456462693721 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9150652985074627, + "pass@2": 0.9429681072981553, + "pass@4": 0.9625132151391523, + "pass@8": 0.9756780202977416, + "pass@16": 0.9835347744234902, + "pass@32": 0.9882837642926804, + "pass@64": 0.9915682030867851, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9854464285714286, + "pass@2": 0.988620992688414, + "pass@4": 0.9901797683325299, + "pass@8": 0.991344744113091, + "pass@16": 0.9925872694705262, + "pass@32": 0.9936111354275262, + "pass@64": 0.9942033681695974, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9927341472303207, + "pass@2": 0.9970397115539129, + "pass@4": 0.9990276626318211, + "pass@8": 0.9997711384831617, + "pass@16": 0.9999856655983261, + "pass@32": 0.9999999740942677, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9947459795321637, + "pass@2": 0.9957608095040752, + "pass@4": 0.9967567209252353, + "pass@8": 0.997803074735884, + "pass@16": 0.9986958293792437, + "pass@32": 0.9995001838500145, + "pass@64": 0.9999595798278741, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9498851102941176, + "pass@2": 0.9584327234830942, + "pass@4": 0.9644861882154439, + "pass@8": 0.968570720112799, + "pass@16": 0.9719730560084113, + "pass@32": 0.9754725300929893, + "pass@64": 0.979253814615793, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9716581230283912, + "pass@2": 0.979230963883852, + "pass@4": 0.9837603316146996, + "pass@8": 0.9871485365512419, + "pass@16": 0.9892880094924388, + "pass@32": 0.9909373526903568, + "pass@64": 0.992720534533528, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9692197389240507, + "pass@2": 0.9778177327319845, + "pass@4": 0.9829445342275255, + "pass@8": 0.9867073769830967, + "pass@16": 0.9900901740260195, + "pass@32": 0.9932564114680369, + "pass@64": 0.995770682007257, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9474474474474475, + "pass@2": 0.96165444479913, + "pass@4": 0.9723225123323547, + "pass@8": 0.9815041002513741, + "pass@16": 0.9893463293362276, + "pass@32": 0.9950770481017992, + "pass@64": 0.9981847806163591, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9587268890675241, + "pass@2": 0.9757762456642275, + "pass@4": 0.983839193030935, + "pass@8": 0.9880694906116392, + "pass@16": 0.9909492939632827, + "pass@32": 0.9929625698571264, + "pass@64": 0.9947843180953608, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.905952380952381, + "pass@2": 0.9413335051868518, + "pass@4": 0.960290828824968, + "pass@8": 0.9709857429670468, + "pass@16": 0.9774405273749097, + "pass@32": 0.9817948412796146, + "pass@64": 0.9855039352653046, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8351490825688074, + "pass@2": 0.8865521683642764, + "pass@4": 0.9158329351606272, + "pass@8": 0.93677803817362, + "pass@16": 0.9529807416222166, + "pass@32": 0.9633928267867302, + "pass@64": 0.969689293803253, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6290991512345679, + "pass@2": 0.7196005881209291, + "pass@4": 0.7843625041661457, + "pass@8": 0.8298234719287071, + "pass@16": 0.8622934266667467, + "pass@32": 0.887627567345078, + "pass@64": 0.9068883538882164, + "pass@128": 0.9228395061728395 + }, + "14": { + "pass@1": 0.517665059347181, + "pass@2": 0.6087565568120749, + "pass@4": 0.684754727802941, + "pass@8": 0.7437021459214515, + "pass@16": 0.787872327619222, + "pass@32": 0.820800001270744, + "pass@64": 0.8461369420166132, + "pass@128": 0.8635014836795252 + }, + "15": { + "pass@1": 0.3942901234567901, + "pass@2": 0.4890691831923788, + "pass@4": 0.5698684924222436, + "pass@8": 0.6340819192059179, + "pass@16": 0.6860708953608408, + "pass@32": 0.731478750850336, + "pass@64": 0.7721371133096265, + "pass@128": 0.808641975308642 + }, + "16": { + "pass@1": 0.29314159292035397, + "pass@2": 0.3814883689870623, + "pass@4": 0.46538661760200317, + "pass@8": 0.5397500092692332, + "pass@16": 0.6028619868483489, + "pass@32": 0.6595435053360812, + "pass@64": 0.7098720418701685, + "pass@128": 0.7463126843657817 + }, + "17": { + "pass@1": 0.23444559487951808, + "pass@2": 0.30919964306043063, + "pass@4": 0.37583967845510263, + "pass@8": 0.4334315980436936, + "pass@16": 0.48871885354179795, + "pass@32": 0.5457485752789358, + "pass@64": 0.6028672476278413, + "pass@128": 0.6536144578313253 + }, + "18": { + "pass@1": 0.21048387096774193, + "pass@2": 0.2815686912623825, + "pass@4": 0.34966251466550535, + "pass@8": 0.41016769779492335, + "pass@16": 0.46298474128011696, + "pass@32": 0.5089164087981424, + "pass@64": 0.5518706752579567, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.17015200151057402, + "pass@2": 0.23729987867830718, + "pass@4": 0.3032997266013954, + "pass@8": 0.36506129258690445, + "pass@16": 0.42155961414972953, + "pass@32": 0.4742200902464383, + "pass@64": 0.5248506977503865, + "pass@128": 0.5709969788519638 + }, + "20": { + "pass@1": 0.16295983483483484, + "pass@2": 0.22672450994301377, + "pass@4": 0.2945440325026939, + "pass@8": 0.35988728724728264, + "pass@16": 0.4203472111206872, + "pass@32": 0.4770773485227238, + "pass@64": 0.5301250956398212, + "pass@128": 0.5765765765765766 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..016ac06c793bee8a883ec7524c1fa0273e0897e1 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffda0fba780922d1325dca30094a37946bab378b2bf3e00243675c7f3778b039 +size 1789772232 diff --git a/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json b/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6e46a92cd7fb342fe18a0df8921d447b74ec81c7 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op11-14", + "id": { + "answer_accuracy": 0.9905703125, + "avg_response_len": 199.1858359375, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9905703125, + "pass@2": 0.9931240567585284, + "pass@4": 0.9946121005395152, + "pass@8": 0.9956282392841028, + "pass@16": 0.9963701188054171, + "pass@32": 0.996918025082482, + "pass@64": 0.9973719313555771, + "pass@128": 0.9977777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.942828125, + "pass@2": 0.9543673720472442, + "pass@4": 0.9623778480502434, + "pass@8": 0.9682502041026957, + "pass@16": 0.972860979589154, + "pass@32": 0.9767782353474479, + "pass@64": 0.9804701699275815, + "pass@128": 0.984 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999953125, + "pass@2": 0.9999992618110236, + "pass@4": 0.9999999999062618, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998390625, + "pass@2": 0.9988737696850394, + "pass@4": 0.9993528392388451, + "pass@8": 0.9997678683581743, + "pass@16": 0.9999654222866098, + "pass@32": 0.9999993776062154, + "pass@64": 0.9999999999893633, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933984375, + "pass@2": 0.9961841781496062, + "pass@4": 0.9975373475815523, + "pass@8": 0.9983680305456933, + "pass@16": 0.9990184921424964, + "pass@32": 0.9995291759985511, + "pass@64": 0.9998772846233093, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99665625, + "pass@2": 0.9977063238188978, + "pass@4": 0.9982420018747656, + "pass@8": 0.9986003427777526, + "pass@16": 0.9988616968167805, + "pass@32": 0.9989862397601972, + "pass@64": 0.998999972481556, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9970390625, + "pass@2": 0.9983011811023623, + "pass@4": 0.9990445734908138, + "pass@8": 0.9995488522357531, + "pass@16": 0.9998741388348003, + "pass@32": 0.9999900821873651, + "pass@64": 0.9999999879305058, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941171875, + "pass@2": 0.9971940206692916, + "pass@4": 0.9985053151481066, + "pass@8": 0.9989375417785472, + "pass@16": 0.9989989424149476, + "pass@32": 0.9989999998940483, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.992765625, + "pass@2": 0.9954905265748032, + "pass@4": 0.9964489795650546, + "pass@8": 0.9971813137583089, + "pass@16": 0.9977513971639642, + "pass@32": 0.9979791149485119, + "pass@64": 0.9979999672478784, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.942828125, + "2": 0.999984375, + "3": 0.999953125, + "4": 0.998390625, + "5": 0.9933984375, + "6": 0.99665625, + "7": 0.9970390625, + "8": 0.9941171875, + "9": 0.992765625 + }, + "per_op_avg_response_len": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375744, + "answer_accuracy": 0.9913880445795339, + "avg_response_len": 189.61989720533603, + "resp_tokens_sum": 71867458, + "pass_at_k": { + "pass@1": 0.9913880445795339, + "pass@2": 0.9933423381917686, + "pass@4": 0.9944101428494183, + "pass@8": 0.995316754918926, + "pass@16": 0.9962039266427511, + "pass@32": 0.9970168336391735, + "pass@64": 0.9977520459936974, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9453585914454278, + "pass@2": 0.9535728491626602, + "pass@4": 0.95873854180838, + "pass@8": 0.9629986555885872, + "pass@16": 0.9679666317812168, + "pass@32": 0.9740341139349422, + "pass@64": 0.9803653794455086, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9971923828125, + "pass@2": 0.9978869340551182, + "pass@4": 0.9985996801181102, + "pass@8": 0.9993905663162155, + "pass@16": 0.9998951809167032, + "pass@32": 0.9999980558628433, + "pass@64": 0.9999999999667597, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949900793650793, + "pass@2": 0.9973585333083365, + "pass@4": 0.9983299611358104, + "pass@8": 0.9991563945664312, + "pass@16": 0.9997971079638622, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391617, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971229416167665, + "pass@2": 0.9976215138620398, + "pass@4": 0.9981272295117304, + "pass@8": 0.998847506991621, + "pass@16": 0.9995863571069055, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964866863905325, + "pass@2": 0.9979623363462704, + "pass@4": 0.9990334570789599, + "pass@8": 0.9997250598303445, + "pass@16": 0.9999790093720811, + "pass@32": 0.9999999405368915, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9981971153846154, + "pass@2": 0.9998284657026045, + "pass@4": 0.999995967751627, + "pass@8": 0.9999999975724242, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.9973813352556282, + "pass@4": 0.9982075685081618, + "pass@8": 0.9989154500291274, + "pass@16": 0.9996108261180917, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386609, + "answer_accuracy": 0.989640502129751, + "avg_response_len": 197.2131773222477, + "resp_tokens_sum": 77042511, + "pass_at_k": { + "pass@1": 0.989640502129751, + "pass@2": 0.9919891258217151, + "pass@4": 0.993503951013332, + "pass@8": 0.9945439716423881, + "pass@16": 0.99522975394596, + "pass@32": 0.9956609074616072, + "pass@64": 0.9959856221204596, + "pass@128": 0.9963958060288336 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9309815950920245, + "pass@2": 0.9449852966040289, + "pass@4": 0.9559561028797782, + "pass@8": 0.9638793675635439, + "pass@16": 0.9687407182381751, + "pass@32": 0.9718304058076516, + "pass@64": 0.9746895660017418, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99375, + "pass@2": 0.9960658450302408, + "pass@4": 0.9975695505453124, + "pass@8": 0.9984974444758158, + "pass@16": 0.9992768151012691, + "pass@32": 0.9998558695467676, + "pass@64": 0.999998064534979, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964183381088825, + "pass@2": 0.9970275026509938, + "pass@4": 0.9971312728100964, + "pass@8": 0.9971346683194743, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9973582008670521, + "pass@2": 0.9977495192526513, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936839788732394, + "pass@2": 0.9963724769879118, + "pass@4": 0.9971180363018003, + "pass@8": 0.9971826215324542, + "pass@16": 0.9971830985804359, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9938524771323495, + "pass@4": 0.9940081237973997, + "pass@8": 0.9940119737766329, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378784, + "answer_accuracy": 0.9907097422162705, + "avg_response_len": 210.68409461834617, + "resp_tokens_sum": 80552114, + "pass_at_k": { + "pass@1": 0.9907097422162705, + "pass@2": 0.9940673034593474, + "pass@4": 0.995944564240317, + "pass@8": 0.9970448747202597, + "pass@16": 0.9977000446657175, + "pass@32": 0.9981045503059652, + "pass@64": 0.9984116020425892, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9517957089552239, + "pass@2": 0.9643014014572805, + "pass@4": 0.9723098293683435, + "pass@8": 0.9778178687535791, + "pass@16": 0.9818233351333638, + "pass@32": 0.9843700251647998, + "pass@64": 0.9862015157581675, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998861151603499, + "pass@2": 0.99999784784555, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979669225146199, + "pass@2": 0.9986840666298292, + "pass@4": 0.9994179578978066, + "pass@8": 0.9998914828566827, + "pass@16": 0.999996971910131, + "pass@32": 0.9999999992108352, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915670955882353, + "pass@2": 0.9952162459471977, + "pass@4": 0.996770337898939, + "pass@8": 0.9975063173915031, + "pass@16": 0.9980350056586529, + "pass@32": 0.9987689956690733, + "pass@64": 0.9996410375173692, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964264589905363, + "pass@2": 0.9985430276459921, + "pass@4": 0.9995857823568582, + "pass@8": 0.9999535526784058, + "pass@16": 0.9999995379907698, + "pass@32": 0.9999999999901541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972804588607594, + "pass@2": 0.9992676511262834, + "pass@4": 0.9999203209764919, + "pass@8": 0.9999990565168133, + "pass@16": 0.9999999999228011, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9907563813813813, + "pass@2": 0.99560153164976, + "pass@4": 0.998587598565945, + "pass@8": 0.9998129486244137, + "pass@16": 0.999996824080759, + "pass@32": 0.9999999996818268, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9906300241157556, + "pass@2": 0.9950913993467858, + "pass@4": 0.9970629562582811, + "pass@8": 0.9986055619182425, + "pass@16": 0.99964486782007, + "pass@32": 0.9999770906276318, + "pass@64": 0.9999999831714547, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6678171875, + "avg_response_len": 367.13116015625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6678171875, + "pass@2": 0.7378225639763673, + "pass@4": 0.7893652439351264, + "pass@8": 0.8270737370351559, + "pass@16": 0.8550486463709078, + "pass@32": 0.8767224088959323, + "pass@64": 0.8945746837166629, + "pass@128": 0.9096 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.963046875, + "pass@2": 0.9739500492125989, + "pass@4": 0.9801725765841772, + "pass@8": 0.9841731737862502, + "pass@16": 0.986679806459084, + "pass@32": 0.9887283379415103, + "pass@64": 0.9903890917527084, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9079453125, + "pass@2": 0.9249128937007879, + "pass@4": 0.9345272270341208, + "pass@8": 0.9416891861472778, + "pass@16": 0.9476419112166174, + "pass@32": 0.9523055926849554, + "pass@64": 0.9562749449531718, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.8557421875, + "pass@2": 0.8929637057086628, + "pass@4": 0.9139262857142856, + "pass@8": 0.9291272854564019, + "pass@16": 0.9415987007431647, + "pass@32": 0.9518761615883907, + "pass@64": 0.9606307335381308, + "pass@128": 0.968 + }, + "14": { + "pass@1": 0.837, + "pass@2": 0.8863106545275599, + "pass@4": 0.9137613407386579, + "pass@8": 0.9312510179501858, + "pass@16": 0.9439751905099076, + "pass@32": 0.9542164522470538, + "pass@64": 0.9626050622049743, + "pass@128": 0.969 + }, + "15": { + "pass@1": 0.77975, + "pass@2": 0.8448986220472454, + "pass@4": 0.8857677772778397, + "pass@8": 0.9115909825900337, + "pass@16": 0.9277541320088688, + "pass@32": 0.9388842588529385, + "pass@64": 0.9474485358545235, + "pass@128": 0.954 + }, + "16": { + "pass@1": 0.6545625, + "pass@2": 0.7510269438976384, + "pass@4": 0.8116648335208099, + "pass@8": 0.8497649349116291, + "pass@16": 0.8757048434440007, + "pass@32": 0.8956257423100381, + "pass@64": 0.9124470610164344, + "pass@128": 0.925 + }, + "17": { + "pass@1": 0.55796875, + "pass@2": 0.6625004921259854, + "pass@4": 0.7365261226096746, + "pass@8": 0.7870371231754127, + "pass@16": 0.8224065318779613, + "pass@32": 0.8483149142099418, + "pass@64": 0.8693774322654676, + "pass@128": 0.888 + }, + "18": { + "pass@1": 0.480296875, + "pass@2": 0.5879370078740163, + "pass@4": 0.6728501369516315, + "pass@8": 0.7364498415117591, + "pass@16": 0.7835286822486924, + "pass@32": 0.8195105952369934, + "pass@64": 0.8482734816022467, + "pass@128": 0.872 + }, + "19": { + "pass@1": 0.33475, + "pass@2": 0.44351464074803204, + "pass@4": 0.5406705727409072, + "pass@8": 0.6195248806878444, + "pass@16": 0.6802573730953311, + "pass@32": 0.7279540135363097, + "pass@64": 0.7678995342731354, + "pass@128": 0.803 + }, + "20": { + "pass@1": 0.307109375, + "pass@2": 0.4102106299212602, + "pass@4": 0.503785566179227, + "pass@8": 0.5801289441348498, + "pass@16": 0.6409392921053366, + "pass@32": 0.6898080203511475, + "pass@64": 0.7304009597057819, + "pass@128": 0.765 + } + }, + "per_op_accuracy": { + "11": 0.963046875, + "12": 0.9079453125, + "13": 0.8557421875, + "14": 0.837, + "15": 0.77975, + "16": 0.6545625, + "17": 0.55796875, + "18": 0.480296875, + "19": 0.33475, + "20": 0.307109375 + }, + "per_op_avg_response_len": { + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 290947, + "answer_accuracy": 0.6754898774145617, + "avg_response_len": 349.3103222511144, + "resp_tokens_sum": 150454942, + "pass_at_k": { + "pass@1": 0.6754898774145617, + "pass@2": 0.7447737390459964, + "pass@4": 0.7952060179698968, + "pass@8": 0.8316698977934396, + "pass@16": 0.8584897543756141, + "pass@32": 0.8792215528888129, + "pass@64": 0.8964337939825147, + "pass@128": 0.9105497771173848 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9636597025495751, + "pass@2": 0.9756313990319198, + "pass@4": 0.9833490821614721, + "pass@8": 0.9890875592461815, + "pass@16": 0.9928681364142148, + "pass@32": 0.9954215669121044, + "pass@64": 0.9969126772151184, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9084348516949152, + "pass@2": 0.9271233011922239, + "pass@4": 0.9376102685257564, + "pass@8": 0.9448320171203957, + "pass@16": 0.9500880057183615, + "pass@32": 0.9533178342586844, + "pass@64": 0.9561329927476948, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8727468652037618, + "pass@2": 0.9105426036087183, + "pass@4": 0.9320865086809287, + "pass@8": 0.9476093057668198, + "pass@16": 0.9599091371962115, + "pass@32": 0.9694213182908559, + "pass@64": 0.9768613994766613, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8451129283489096, + "pass@2": 0.8927793926460124, + "pass@4": 0.9188406133812713, + "pass@8": 0.9352618975128104, + "pass@16": 0.9480306260712734, + "pass@32": 0.9589256381658317, + "pass@64": 0.9683198089214712, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7921107297687862, + "pass@2": 0.8578609303172365, + "pass@4": 0.8997546105147255, + "pass@8": 0.9264640318143393, + "pass@16": 0.9428917374738313, + "pass@32": 0.952393511786133, + "pass@64": 0.9574741921523455, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.6568800403225806, + "pass@2": 0.7477683674117346, + "pass@4": 0.8092675075897764, + "pass@8": 0.8516551322611292, + "pass@16": 0.8812944796205525, + "pass@32": 0.902773745388599, + "pass@64": 0.9194705540558217, + "pass@128": 0.932258064516129 + }, + "17": { + "pass@1": 0.5657514450867052, + "pass@2": 0.6691021687679213, + "pass@4": 0.739191088198512, + "pass@8": 0.7856207032319886, + "pass@16": 0.8183619373072404, + "pass@32": 0.8430109049522699, + "pass@64": 0.8637531249758412, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.45412946428571427, + "pass@2": 0.5638899043869511, + "pass@4": 0.6530802185441107, + "pass@8": 0.720726478211569, + "pass@16": 0.7704510468029769, + "pass@32": 0.807289056099719, + "pass@64": 0.8378334239926626, + "pass@128": 0.8657142857142858 + }, + "19": { + "pass@1": 0.36369181681681684, + "pass@2": 0.4703841390603205, + "pass@4": 0.5581976540657639, + "pass@8": 0.627149829400672, + "pass@16": 0.6817550885633032, + "pass@32": 0.7263554189791901, + "pass@64": 0.763762224924364, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.32453547297297297, + "pass@2": 0.4275736119978244, + "pass@4": 0.5169445100330928, + "pass@8": 0.585449717921024, + "pass@16": 0.6372318334285229, + "pass@32": 0.6821752409020172, + "pass@64": 0.7234653885399501, + "pass@128": 0.7537537537537538 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 291147, + "answer_accuracy": 0.6763562109723461, + "avg_response_len": 362.8061301293488, + "resp_tokens_sum": 156174978, + "pass_at_k": { + "pass@1": 0.6763562109723461, + "pass@2": 0.7465801048815182, + "pass@4": 0.7976332155904704, + "pass@8": 0.8348140204603103, + "pass@16": 0.8624812646405657, + "pass@32": 0.883998466128224, + "pass@64": 0.9017876930240121, + "pass@128": 0.9170383586083853 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9630082831325302, + "pass@2": 0.9713395906460487, + "pass@4": 0.9757507891521089, + "pass@8": 0.9784955531653872, + "pass@16": 0.9805383638724173, + "pass@32": 0.9824747306648515, + "pass@64": 0.9841054553733752, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9007885971786834, + "pass@2": 0.915418739046726, + "pass@4": 0.924029935129582, + "pass@8": 0.9303735376408099, + "pass@16": 0.9353211411280159, + "pass@32": 0.939039317249673, + "pass@64": 0.9419950990841026, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.8611913515406162, + "pass@2": 0.8988553590727634, + "pass@4": 0.918313749121695, + "pass@8": 0.9307861443620137, + "pass@16": 0.9400313356135247, + "pass@32": 0.9484692530915426, + "pass@64": 0.9581816741685208, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8469709429824561, + "pass@2": 0.8980640886172121, + "pass@4": 0.9245809564374623, + "pass@8": 0.9405203926008627, + "pass@16": 0.9525469091473052, + "pass@32": 0.963199459193292, + "pass@64": 0.9716362530225532, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7924715909090909, + "pass@2": 0.861036670842281, + "pass@4": 0.9029049320539471, + "pass@8": 0.9281268028368013, + "pass@16": 0.9426543876503707, + "pass@32": 0.9525476884583312, + "pass@64": 0.9611302965599587, + "pass@128": 0.9696969696969697 + }, + "16": { + "pass@1": 0.6614583333333334, + "pass@2": 0.757938160374184, + "pass@4": 0.8144365875205767, + "pass@8": 0.8475187224277707, + "pass@16": 0.8701097260916296, + "pass@32": 0.8890719227083439, + "pass@64": 0.9063072562149335, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5830260093167702, + "pass@2": 0.6846121986110436, + "pass@4": 0.7533166365656156, + "pass@8": 0.8007323980276663, + "pass@16": 0.8367999826994603, + "pass@32": 0.8654849657428961, + "pass@64": 0.8891736360856827, + "pass@128": 0.906832298136646 + }, + "18": { + "pass@1": 0.5104779411764706, + "pass@2": 0.615626447429365, + "pass@4": 0.6993681317408854, + "pass@8": 0.7636960252154447, + "pass@16": 0.8126314154686012, + "pass@32": 0.8493046683312366, + "pass@64": 0.8760405951252511, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.33535621279761907, + "pass@2": 0.450107066694788, + "pass@4": 0.5589451405069903, + "pass@8": 0.6479033713249467, + "pass@16": 0.7114656294943115, + "pass@32": 0.7547122095604747, + "pass@64": 0.7871870069795173, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.30761133982035926, + "pass@2": 0.4103107760856239, + "pass@4": 0.5020851449519407, + "pass@8": 0.577641649195555, + "pass@16": 0.640643393523646, + "pass@32": 0.6937003584791263, + "pass@64": 0.7400380698580312, + "pass@128": 0.7844311377245509 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 272712, + "answer_accuracy": 0.6511499083129584, + "avg_response_len": 389.90383605210883, + "resp_tokens_sum": 163297965, + "pass_at_k": { + "pass@1": 0.6511499083129584, + "pass@2": 0.7216727124564459, + "pass@4": 0.7748605393801656, + "pass@8": 0.8143913855344584, + "pass@16": 0.8438703995259332, + "pass@32": 0.8666738147613688, + "pass@64": 0.8852491163739664, + "pass@128": 0.9009779951100244 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9624007936507937, + "pass@2": 0.9748172103487065, + "pass@4": 0.981273297087864, + "pass@8": 0.9846499737188555, + "pass@16": 0.9862178333308695, + "pass@32": 0.9878187753676408, + "pass@64": 0.9897012873390828, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9143969801223242, + "pass@2": 0.9317818572563751, + "pass@4": 0.941430099907236, + "pass@8": 0.9493256684380422, + "pass@16": 0.9570131901299108, + "pass@32": 0.9641514897698327, + "pass@64": 0.9703591098246457, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8329957561728395, + "pass@2": 0.8691643579274818, + "pass@4": 0.8912119784216789, + "pass@8": 0.9091026647516887, + "pass@16": 0.9252978369245227, + "pass@32": 0.9383556719133557, + "pass@64": 0.9473490414410928, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8191533753709199, + "pass@2": 0.8682212201219658, + "pass@4": 0.8979430763847394, + "pass@8": 0.9180236634394026, + "pass@16": 0.9314134024114261, + "pass@32": 0.9406145618745281, + "pass@64": 0.9479964540280973, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.7535927854938271, + "pass@2": 0.8146192554923689, + "pass@4": 0.8533767113022905, + "pass@8": 0.878865995821382, + "pass@16": 0.8964124781429048, + "pass@32": 0.910541254887923, + "pass@64": 0.922806998472301, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.6453032817109144, + "pass@2": 0.7468509017954611, + "pass@4": 0.8109872092868917, + "pass@8": 0.8503621602906531, + "pass@16": 0.8763865513376642, + "pass@32": 0.8958750335367089, + "pass@64": 0.9123815407896373, + "pass@128": 0.9262536873156342 + }, + "17": { + "pass@1": 0.5255553463855421, + "pass@2": 0.6341747402997818, + "pass@4": 0.7174640033700606, + "pass@8": 0.7752305050970983, + "pass@16": 0.8126617684952755, + "pass@32": 0.8371897052025431, + "pass@64": 0.8560389765187822, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4767389112903226, + "pass@2": 0.5847179006858011, + "pass@4": 0.66608676022352, + "pass@8": 0.7243191147240607, + "pass@16": 0.7663746277687996, + "pass@32": 0.8006317366757167, + "pass@64": 0.829606389878158, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.30501793806646527, + "pass@2": 0.40979077955134763, + "pass@4": 0.5044869689627164, + "pass@8": 0.5830466910339542, + "pass@16": 0.6470709277754211, + "pass@32": 0.7023998688940181, + "pass@64": 0.7524830182120951, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.2891798048048048, + "pass@2": 0.3927472009410985, + "pass@4": 0.49233214992270086, + "pass@8": 0.5773029346421416, + "pass@16": 0.6449435379481746, + "pass@32": 0.6935367729692132, + "pass@64": 0.7276704804486359, + "pass@128": 0.7567567567567568 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8207002467105263, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8207002467105263, + "pass@2": 0.8587548500310641, + "pass@4": 0.8865874391687733, + "pass@8": 0.9069153433636171, + "pass@16": 0.9219903964714486, + "pass@32": 0.9336571744579908, + "pass@64": 0.9432681168087804, + "pass@128": 0.9513684210526315 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.942828125, + "pass@2": 0.9543673720472442, + "pass@4": 0.9623778480502434, + "pass@8": 0.9682502041026957, + "pass@16": 0.972860979589154, + "pass@32": 0.9767782353474479, + "pass@64": 0.9804701699275815, + "pass@128": 0.984 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999953125, + "pass@2": 0.9999992618110236, + "pass@4": 0.9999999999062618, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998390625, + "pass@2": 0.9988737696850394, + "pass@4": 0.9993528392388451, + "pass@8": 0.9997678683581743, + "pass@16": 0.9999654222866098, + "pass@32": 0.9999993776062154, + "pass@64": 0.9999999999893633, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933984375, + "pass@2": 0.9961841781496062, + "pass@4": 0.9975373475815523, + "pass@8": 0.9983680305456933, + "pass@16": 0.9990184921424964, + "pass@32": 0.9995291759985511, + "pass@64": 0.9998772846233093, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99665625, + "pass@2": 0.9977063238188978, + "pass@4": 0.9982420018747656, + "pass@8": 0.9986003427777526, + "pass@16": 0.9988616968167805, + "pass@32": 0.9989862397601972, + "pass@64": 0.998999972481556, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9970390625, + "pass@2": 0.9983011811023623, + "pass@4": 0.9990445734908138, + "pass@8": 0.9995488522357531, + "pass@16": 0.9998741388348003, + "pass@32": 0.9999900821873651, + "pass@64": 0.9999999879305058, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941171875, + "pass@2": 0.9971940206692916, + "pass@4": 0.9985053151481066, + "pass@8": 0.9989375417785472, + "pass@16": 0.9989989424149476, + "pass@32": 0.9989999998940483, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.992765625, + "pass@2": 0.9954905265748032, + "pass@4": 0.9964489795650546, + "pass@8": 0.9971813137583089, + "pass@16": 0.9977513971639642, + "pass@32": 0.9979791149485119, + "pass@64": 0.9979999672478784, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.963046875, + "pass@2": 0.9739500492125989, + "pass@4": 0.9801725765841772, + "pass@8": 0.9841731737862502, + "pass@16": 0.986679806459084, + "pass@32": 0.9887283379415103, + "pass@64": 0.9903890917527084, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9079453125, + "pass@2": 0.9249128937007879, + "pass@4": 0.9345272270341208, + "pass@8": 0.9416891861472778, + "pass@16": 0.9476419112166174, + "pass@32": 0.9523055926849554, + "pass@64": 0.9562749449531718, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.8557421875, + "pass@2": 0.8929637057086628, + "pass@4": 0.9139262857142856, + "pass@8": 0.9291272854564019, + "pass@16": 0.9415987007431647, + "pass@32": 0.9518761615883907, + "pass@64": 0.9606307335381308, + "pass@128": 0.968 + }, + "14": { + "pass@1": 0.837, + "pass@2": 0.8863106545275599, + "pass@4": 0.9137613407386579, + "pass@8": 0.9312510179501858, + "pass@16": 0.9439751905099076, + "pass@32": 0.9542164522470538, + "pass@64": 0.9626050622049743, + "pass@128": 0.969 + }, + "15": { + "pass@1": 0.77975, + "pass@2": 0.8448986220472454, + "pass@4": 0.8857677772778397, + "pass@8": 0.9115909825900337, + "pass@16": 0.9277541320088688, + "pass@32": 0.9388842588529385, + "pass@64": 0.9474485358545235, + "pass@128": 0.954 + }, + "16": { + "pass@1": 0.6545625, + "pass@2": 0.7510269438976384, + "pass@4": 0.8116648335208099, + "pass@8": 0.8497649349116291, + "pass@16": 0.8757048434440007, + "pass@32": 0.8956257423100381, + "pass@64": 0.9124470610164344, + "pass@128": 0.925 + }, + "17": { + "pass@1": 0.55796875, + "pass@2": 0.6625004921259854, + "pass@4": 0.7365261226096746, + "pass@8": 0.7870371231754127, + "pass@16": 0.8224065318779613, + "pass@32": 0.8483149142099418, + "pass@64": 0.8693774322654676, + "pass@128": 0.888 + }, + "18": { + "pass@1": 0.480296875, + "pass@2": 0.5879370078740163, + "pass@4": 0.6728501369516315, + "pass@8": 0.7364498415117591, + "pass@16": 0.7835286822486924, + "pass@32": 0.8195105952369934, + "pass@64": 0.8482734816022467, + "pass@128": 0.872 + }, + "19": { + "pass@1": 0.33475, + "pass@2": 0.44351464074803204, + "pass@4": 0.5406705727409072, + "pass@8": 0.6195248806878444, + "pass@16": 0.6802573730953311, + "pass@32": 0.7279540135363097, + "pass@64": 0.7678995342731354, + "pass@128": 0.803 + }, + "20": { + "pass@1": 0.307109375, + "pass@2": 0.4102106299212602, + "pass@4": 0.503785566179227, + "pass@8": 0.5801289441348498, + "pass@16": 0.6409392921053366, + "pass@32": 0.6898080203511475, + "pass@64": 0.7304009597057819, + "pass@128": 0.765 + } + }, + "per_op_accuracy": { + "10": 0.942828125, + "2": 0.999984375, + "3": 0.999953125, + "4": 0.998390625, + "5": 0.9933984375, + "6": 0.99665625, + "7": 0.9970390625, + "8": 0.9941171875, + "9": 0.992765625, + "11": 0.963046875, + "12": 0.9079453125, + "13": 0.8557421875, + "14": 0.837, + "15": 0.77975, + "16": 0.6545625, + "17": 0.55796875, + "18": 0.480296875, + "19": 0.33475, + "20": 0.307109375 + }, + "per_op_avg_response_len": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375, + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375, + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 666691, + "answer_accuracy": 0.8233517922067657, + "avg_response_len": 274.56429813468225, + "resp_tokens_sum": 222322400, + "pass_at_k": { + "pass@1": 0.8233517922067657, + "pass@2": 0.8611208180960408, + "pass@4": 0.8884471519832107, + "pass@8": 0.9082678023063252, + "pass@16": 0.9229493914421676, + "pass@32": 0.9343577884724095, + "pass@64": 0.9438576549065015, + "pass@128": 0.9516282010749288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9453585914454278, + "pass@2": 0.9535728491626602, + "pass@4": 0.95873854180838, + "pass@8": 0.9629986555885872, + "pass@16": 0.9679666317812168, + "pass@32": 0.9740341139349422, + "pass@64": 0.9803653794455086, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9971923828125, + "pass@2": 0.9978869340551182, + "pass@4": 0.9985996801181102, + "pass@8": 0.9993905663162155, + "pass@16": 0.9998951809167032, + "pass@32": 0.9999980558628433, + "pass@64": 0.9999999999667597, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949900793650793, + "pass@2": 0.9973585333083365, + "pass@4": 0.9983299611358104, + "pass@8": 0.9991563945664312, + "pass@16": 0.9997971079638622, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391617, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971229416167665, + "pass@2": 0.9976215138620398, + "pass@4": 0.9981272295117304, + "pass@8": 0.998847506991621, + "pass@16": 0.9995863571069055, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964866863905325, + "pass@2": 0.9979623363462704, + "pass@4": 0.9990334570789599, + "pass@8": 0.9997250598303445, + "pass@16": 0.9999790093720811, + "pass@32": 0.9999999405368915, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9981971153846154, + "pass@2": 0.9998284657026045, + "pass@4": 0.999995967751627, + "pass@8": 0.9999999975724242, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.9973813352556282, + "pass@4": 0.9982075685081618, + "pass@8": 0.9989154500291274, + "pass@16": 0.9996108261180917, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9636597025495751, + "pass@2": 0.9756313990319198, + "pass@4": 0.9833490821614721, + "pass@8": 0.9890875592461815, + "pass@16": 0.9928681364142148, + "pass@32": 0.9954215669121044, + "pass@64": 0.9969126772151184, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9084348516949152, + "pass@2": 0.9271233011922239, + "pass@4": 0.9376102685257564, + "pass@8": 0.9448320171203957, + "pass@16": 0.9500880057183615, + "pass@32": 0.9533178342586844, + "pass@64": 0.9561329927476948, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8727468652037618, + "pass@2": 0.9105426036087183, + "pass@4": 0.9320865086809287, + "pass@8": 0.9476093057668198, + "pass@16": 0.9599091371962115, + "pass@32": 0.9694213182908559, + "pass@64": 0.9768613994766613, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8451129283489096, + "pass@2": 0.8927793926460124, + "pass@4": 0.9188406133812713, + "pass@8": 0.9352618975128104, + "pass@16": 0.9480306260712734, + "pass@32": 0.9589256381658317, + "pass@64": 0.9683198089214712, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7921107297687862, + "pass@2": 0.8578609303172365, + "pass@4": 0.8997546105147255, + "pass@8": 0.9264640318143393, + "pass@16": 0.9428917374738313, + "pass@32": 0.952393511786133, + "pass@64": 0.9574741921523455, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.6568800403225806, + "pass@2": 0.7477683674117346, + "pass@4": 0.8092675075897764, + "pass@8": 0.8516551322611292, + "pass@16": 0.8812944796205525, + "pass@32": 0.902773745388599, + "pass@64": 0.9194705540558217, + "pass@128": 0.932258064516129 + }, + "17": { + "pass@1": 0.5657514450867052, + "pass@2": 0.6691021687679213, + "pass@4": 0.739191088198512, + "pass@8": 0.7856207032319886, + "pass@16": 0.8183619373072404, + "pass@32": 0.8430109049522699, + "pass@64": 0.8637531249758412, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.45412946428571427, + "pass@2": 0.5638899043869511, + "pass@4": 0.6530802185441107, + "pass@8": 0.720726478211569, + "pass@16": 0.7704510468029769, + "pass@32": 0.807289056099719, + "pass@64": 0.8378334239926626, + "pass@128": 0.8657142857142858 + }, + "19": { + "pass@1": 0.36369181681681684, + "pass@2": 0.4703841390603205, + "pass@4": 0.5581976540657639, + "pass@8": 0.627149829400672, + "pass@16": 0.6817550885633032, + "pass@32": 0.7263554189791901, + "pass@64": 0.763762224924364, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.32453547297297297, + "pass@2": 0.4275736119978244, + "pass@4": 0.5169445100330928, + "pass@8": 0.585449717921024, + "pass@16": 0.6372318334285229, + "pass@32": 0.6821752409020172, + "pass@64": 0.7234653885399501, + "pass@128": 0.7537537537537538 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 677756, + "answer_accuracy": 0.8254043257989088, + "avg_response_len": 284.02363722720185, + "resp_tokens_sum": 233217489, + "pass_at_k": { + "pass@1": 0.8254043257989088, + "pass@2": 0.8633358853818162, + "pass@4": 0.8908206644619501, + "pass@8": 0.9108071320749088, + "pass@16": 0.9256376776351245, + "pass@32": 0.9371229822544132, + "pass@64": 0.9466032938973412, + "pass@128": 0.9547934528448948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9309815950920245, + "pass@2": 0.9449852966040289, + "pass@4": 0.9559561028797782, + "pass@8": 0.9638793675635439, + "pass@16": 0.9687407182381751, + "pass@32": 0.9718304058076516, + "pass@64": 0.9746895660017418, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99375, + "pass@2": 0.9960658450302408, + "pass@4": 0.9975695505453124, + "pass@8": 0.9984974444758158, + "pass@16": 0.9992768151012691, + "pass@32": 0.9998558695467676, + "pass@64": 0.999998064534979, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964183381088825, + "pass@2": 0.9970275026509938, + "pass@4": 0.9971312728100964, + "pass@8": 0.9971346683194743, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9973582008670521, + "pass@2": 0.9977495192526513, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936839788732394, + "pass@2": 0.9963724769879118, + "pass@4": 0.9971180363018003, + "pass@8": 0.9971826215324542, + "pass@16": 0.9971830985804359, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9938524771323495, + "pass@4": 0.9940081237973997, + "pass@8": 0.9940119737766329, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9630082831325302, + "pass@2": 0.9713395906460487, + "pass@4": 0.9757507891521089, + "pass@8": 0.9784955531653872, + "pass@16": 0.9805383638724173, + "pass@32": 0.9824747306648515, + "pass@64": 0.9841054553733752, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9007885971786834, + "pass@2": 0.915418739046726, + "pass@4": 0.924029935129582, + "pass@8": 0.9303735376408099, + "pass@16": 0.9353211411280159, + "pass@32": 0.939039317249673, + "pass@64": 0.9419950990841026, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.8611913515406162, + "pass@2": 0.8988553590727634, + "pass@4": 0.918313749121695, + "pass@8": 0.9307861443620137, + "pass@16": 0.9400313356135247, + "pass@32": 0.9484692530915426, + "pass@64": 0.9581816741685208, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8469709429824561, + "pass@2": 0.8980640886172121, + "pass@4": 0.9245809564374623, + "pass@8": 0.9405203926008627, + "pass@16": 0.9525469091473052, + "pass@32": 0.963199459193292, + "pass@64": 0.9716362530225532, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7924715909090909, + "pass@2": 0.861036670842281, + "pass@4": 0.9029049320539471, + "pass@8": 0.9281268028368013, + "pass@16": 0.9426543876503707, + "pass@32": 0.9525476884583312, + "pass@64": 0.9611302965599587, + "pass@128": 0.9696969696969697 + }, + "16": { + "pass@1": 0.6614583333333334, + "pass@2": 0.757938160374184, + "pass@4": 0.8144365875205767, + "pass@8": 0.8475187224277707, + "pass@16": 0.8701097260916296, + "pass@32": 0.8890719227083439, + "pass@64": 0.9063072562149335, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5830260093167702, + "pass@2": 0.6846121986110436, + "pass@4": 0.7533166365656156, + "pass@8": 0.8007323980276663, + "pass@16": 0.8367999826994603, + "pass@32": 0.8654849657428961, + "pass@64": 0.8891736360856827, + "pass@128": 0.906832298136646 + }, + "18": { + "pass@1": 0.5104779411764706, + "pass@2": 0.615626447429365, + "pass@4": 0.6993681317408854, + "pass@8": 0.7636960252154447, + "pass@16": 0.8126314154686012, + "pass@32": 0.8493046683312366, + "pass@64": 0.8760405951252511, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.33535621279761907, + "pass@2": 0.450107066694788, + "pass@4": 0.5589451405069903, + "pass@8": 0.6479033713249467, + "pass@16": 0.7114656294943115, + "pass@32": 0.7547122095604747, + "pass@64": 0.7871870069795173, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.30761133982035926, + "pass@2": 0.4103107760856239, + "pass@4": 0.5020851449519407, + "pass@8": 0.577641649195555, + "pass@16": 0.640643393523646, + "pass@32": 0.6937003584791263, + "pass@64": 0.7400380698580312, + "pass@128": 0.7844311377245509 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 651496, + "answer_accuracy": 0.8131989934494328, + "avg_response_len": 304.374299758348, + "resp_tokens_sum": 243850079, + "pass_at_k": { + "pass@1": 0.8131989934494328, + "pass@2": 0.8516683416824594, + "pass@4": 0.8803690842367309, + "pass@8": 0.9015596188301824, + "pass@16": 0.9172829494592375, + "pass@32": 0.9293968706923023, + "pass@64": 0.9392539645433563, + "pass@128": 0.9475954625339511 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9517957089552239, + "pass@2": 0.9643014014572805, + "pass@4": 0.9723098293683435, + "pass@8": 0.9778178687535791, + "pass@16": 0.9818233351333638, + "pass@32": 0.9843700251647998, + "pass@64": 0.9862015157581675, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998861151603499, + "pass@2": 0.99999784784555, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979669225146199, + "pass@2": 0.9986840666298292, + "pass@4": 0.9994179578978066, + "pass@8": 0.9998914828566827, + "pass@16": 0.999996971910131, + "pass@32": 0.9999999992108352, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915670955882353, + "pass@2": 0.9952162459471977, + "pass@4": 0.996770337898939, + "pass@8": 0.9975063173915031, + "pass@16": 0.9980350056586529, + "pass@32": 0.9987689956690733, + "pass@64": 0.9996410375173692, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964264589905363, + "pass@2": 0.9985430276459921, + "pass@4": 0.9995857823568582, + "pass@8": 0.9999535526784058, + "pass@16": 0.9999995379907698, + "pass@32": 0.9999999999901541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972804588607594, + "pass@2": 0.9992676511262834, + "pass@4": 0.9999203209764919, + "pass@8": 0.9999990565168133, + "pass@16": 0.9999999999228011, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9907563813813813, + "pass@2": 0.99560153164976, + "pass@4": 0.998587598565945, + "pass@8": 0.9998129486244137, + "pass@16": 0.999996824080759, + "pass@32": 0.9999999996818268, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9906300241157556, + "pass@2": 0.9950913993467858, + "pass@4": 0.9970629562582811, + "pass@8": 0.9986055619182425, + "pass@16": 0.99964486782007, + "pass@32": 0.9999770906276318, + "pass@64": 0.9999999831714547, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9624007936507937, + "pass@2": 0.9748172103487065, + "pass@4": 0.981273297087864, + "pass@8": 0.9846499737188555, + "pass@16": 0.9862178333308695, + "pass@32": 0.9878187753676408, + "pass@64": 0.9897012873390828, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9143969801223242, + "pass@2": 0.9317818572563751, + "pass@4": 0.941430099907236, + "pass@8": 0.9493256684380422, + "pass@16": 0.9570131901299108, + "pass@32": 0.9641514897698327, + "pass@64": 0.9703591098246457, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8329957561728395, + "pass@2": 0.8691643579274818, + "pass@4": 0.8912119784216789, + "pass@8": 0.9091026647516887, + "pass@16": 0.9252978369245227, + "pass@32": 0.9383556719133557, + "pass@64": 0.9473490414410928, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8191533753709199, + "pass@2": 0.8682212201219658, + "pass@4": 0.8979430763847394, + "pass@8": 0.9180236634394026, + "pass@16": 0.9314134024114261, + "pass@32": 0.9406145618745281, + "pass@64": 0.9479964540280973, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.7535927854938271, + "pass@2": 0.8146192554923689, + "pass@4": 0.8533767113022905, + "pass@8": 0.878865995821382, + "pass@16": 0.8964124781429048, + "pass@32": 0.910541254887923, + "pass@64": 0.922806998472301, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.6453032817109144, + "pass@2": 0.7468509017954611, + "pass@4": 0.8109872092868917, + "pass@8": 0.8503621602906531, + "pass@16": 0.8763865513376642, + "pass@32": 0.8958750335367089, + "pass@64": 0.9123815407896373, + "pass@128": 0.9262536873156342 + }, + "17": { + "pass@1": 0.5255553463855421, + "pass@2": 0.6341747402997818, + "pass@4": 0.7174640033700606, + "pass@8": 0.7752305050970983, + "pass@16": 0.8126617684952755, + "pass@32": 0.8371897052025431, + "pass@64": 0.8560389765187822, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4767389112903226, + "pass@2": 0.5847179006858011, + "pass@4": 0.66608676022352, + "pass@8": 0.7243191147240607, + "pass@16": 0.7663746277687996, + "pass@32": 0.8006317366757167, + "pass@64": 0.829606389878158, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.30501793806646527, + "pass@2": 0.40979077955134763, + "pass@4": 0.5044869689627164, + "pass@8": 0.5830466910339542, + "pass@16": 0.6470709277754211, + "pass@32": 0.7023998688940181, + "pass@64": 0.7524830182120951, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.2891798048048048, + "pass@2": 0.3927472009410985, + "pass@4": 0.49233214992270086, + "pass@8": 0.5773029346421416, + "pass@16": 0.6449435379481746, + "pass@32": 0.6935367729692132, + "pass@64": 0.7276704804486359, + "pass@128": 0.7567567567567568 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ed79c56caee3fc8ddcd2206f4e4ced6ddfb83050 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8240e70e3f31767a356d1233f48142ad393af64f291e3e2a8dd5e290e3951805 +size 2693207340 diff --git a/results/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv b/results/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..b2297da475338d5ed4d78673919ec0f8b1498c44 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op11-14,0.9905703125,0.6678171875,0.8207002467105263,0.0,0.0,0.0,199.1858359375,367.13116015625,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch4-op11-14/summary.json b/results/composition/SDAR-100M-block16-epoch4-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6682b9543009084976a677597dc3031b491f824d --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op11-14", + "id_acc": 0.9905703125, + "ood_acc": 0.6678171875, + "total_acc": 0.8207002467105263, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.1858359375, + "ood_avg_resp_len": 367.13116015625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 666691, + "answer_accuracy": 0.8233517922067657, + "avg_response_len": 274.56429813468225, + "resp_tokens_sum": 222322400, + "pass_at_k": { + "pass@1": 0.8233517922067657, + "pass@2": 0.8611208180960408, + "pass@4": 0.8884471519832107, + "pass@8": 0.9082678023063252, + "pass@16": 0.9229493914421676, + "pass@32": 0.9343577884724095, + "pass@64": 0.9438576549065015, + "pass@128": 0.9516282010749288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9453585914454278, + "pass@2": 0.9535728491626602, + "pass@4": 0.95873854180838, + "pass@8": 0.9629986555885872, + "pass@16": 0.9679666317812168, + "pass@32": 0.9740341139349422, + "pass@64": 0.9803653794455086, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9971923828125, + "pass@2": 0.9978869340551182, + "pass@4": 0.9985996801181102, + "pass@8": 0.9993905663162155, + "pass@16": 0.9998951809167032, + "pass@32": 0.9999980558628433, + "pass@64": 0.9999999999667597, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949900793650793, + "pass@2": 0.9973585333083365, + "pass@4": 0.9983299611358104, + "pass@8": 0.9991563945664312, + "pass@16": 0.9997971079638622, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391617, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971229416167665, + "pass@2": 0.9976215138620398, + "pass@4": 0.9981272295117304, + "pass@8": 0.998847506991621, + "pass@16": 0.9995863571069055, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964866863905325, + "pass@2": 0.9979623363462704, + "pass@4": 0.9990334570789599, + "pass@8": 0.9997250598303445, + "pass@16": 0.9999790093720811, + "pass@32": 0.9999999405368915, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9981971153846154, + "pass@2": 0.9998284657026045, + "pass@4": 0.999995967751627, + "pass@8": 0.9999999975724242, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.9973813352556282, + "pass@4": 0.9982075685081618, + "pass@8": 0.9989154500291274, + "pass@16": 0.9996108261180917, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9636597025495751, + "pass@2": 0.9756313990319198, + "pass@4": 0.9833490821614721, + "pass@8": 0.9890875592461815, + "pass@16": 0.9928681364142148, + "pass@32": 0.9954215669121044, + "pass@64": 0.9969126772151184, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9084348516949152, + "pass@2": 0.9271233011922239, + "pass@4": 0.9376102685257564, + "pass@8": 0.9448320171203957, + "pass@16": 0.9500880057183615, + "pass@32": 0.9533178342586844, + "pass@64": 0.9561329927476948, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8727468652037618, + "pass@2": 0.9105426036087183, + "pass@4": 0.9320865086809287, + "pass@8": 0.9476093057668198, + "pass@16": 0.9599091371962115, + "pass@32": 0.9694213182908559, + "pass@64": 0.9768613994766613, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8451129283489096, + "pass@2": 0.8927793926460124, + "pass@4": 0.9188406133812713, + "pass@8": 0.9352618975128104, + "pass@16": 0.9480306260712734, + "pass@32": 0.9589256381658317, + "pass@64": 0.9683198089214712, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7921107297687862, + "pass@2": 0.8578609303172365, + "pass@4": 0.8997546105147255, + "pass@8": 0.9264640318143393, + "pass@16": 0.9428917374738313, + "pass@32": 0.952393511786133, + "pass@64": 0.9574741921523455, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.6568800403225806, + "pass@2": 0.7477683674117346, + "pass@4": 0.8092675075897764, + "pass@8": 0.8516551322611292, + "pass@16": 0.8812944796205525, + "pass@32": 0.902773745388599, + "pass@64": 0.9194705540558217, + "pass@128": 0.932258064516129 + }, + "17": { + "pass@1": 0.5657514450867052, + "pass@2": 0.6691021687679213, + "pass@4": 0.739191088198512, + "pass@8": 0.7856207032319886, + "pass@16": 0.8183619373072404, + "pass@32": 0.8430109049522699, + "pass@64": 0.8637531249758412, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.45412946428571427, + "pass@2": 0.5638899043869511, + "pass@4": 0.6530802185441107, + "pass@8": 0.720726478211569, + "pass@16": 0.7704510468029769, + "pass@32": 0.807289056099719, + "pass@64": 0.8378334239926626, + "pass@128": 0.8657142857142858 + }, + "19": { + "pass@1": 0.36369181681681684, + "pass@2": 0.4703841390603205, + "pass@4": 0.5581976540657639, + "pass@8": 0.627149829400672, + "pass@16": 0.6817550885633032, + "pass@32": 0.7263554189791901, + "pass@64": 0.763762224924364, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.32453547297297297, + "pass@2": 0.4275736119978244, + "pass@4": 0.5169445100330928, + "pass@8": 0.585449717921024, + "pass@16": 0.6372318334285229, + "pass@32": 0.6821752409020172, + "pass@64": 0.7234653885399501, + "pass@128": 0.7537537537537538 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 677756, + "answer_accuracy": 0.8254043257989088, + "avg_response_len": 284.02363722720185, + "resp_tokens_sum": 233217489, + "pass_at_k": { + "pass@1": 0.8254043257989088, + "pass@2": 0.8633358853818162, + "pass@4": 0.8908206644619501, + "pass@8": 0.9108071320749088, + "pass@16": 0.9256376776351245, + "pass@32": 0.9371229822544132, + "pass@64": 0.9466032938973412, + "pass@128": 0.9547934528448948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9309815950920245, + "pass@2": 0.9449852966040289, + "pass@4": 0.9559561028797782, + "pass@8": 0.9638793675635439, + "pass@16": 0.9687407182381751, + "pass@32": 0.9718304058076516, + "pass@64": 0.9746895660017418, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99375, + "pass@2": 0.9960658450302408, + "pass@4": 0.9975695505453124, + "pass@8": 0.9984974444758158, + "pass@16": 0.9992768151012691, + "pass@32": 0.9998558695467676, + "pass@64": 0.999998064534979, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964183381088825, + "pass@2": 0.9970275026509938, + "pass@4": 0.9971312728100964, + "pass@8": 0.9971346683194743, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9973582008670521, + "pass@2": 0.9977495192526513, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936839788732394, + "pass@2": 0.9963724769879118, + "pass@4": 0.9971180363018003, + "pass@8": 0.9971826215324542, + "pass@16": 0.9971830985804359, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9938524771323495, + "pass@4": 0.9940081237973997, + "pass@8": 0.9940119737766329, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9630082831325302, + "pass@2": 0.9713395906460487, + "pass@4": 0.9757507891521089, + "pass@8": 0.9784955531653872, + "pass@16": 0.9805383638724173, + "pass@32": 0.9824747306648515, + "pass@64": 0.9841054553733752, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9007885971786834, + "pass@2": 0.915418739046726, + "pass@4": 0.924029935129582, + "pass@8": 0.9303735376408099, + "pass@16": 0.9353211411280159, + "pass@32": 0.939039317249673, + "pass@64": 0.9419950990841026, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.8611913515406162, + "pass@2": 0.8988553590727634, + "pass@4": 0.918313749121695, + "pass@8": 0.9307861443620137, + "pass@16": 0.9400313356135247, + "pass@32": 0.9484692530915426, + "pass@64": 0.9581816741685208, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8469709429824561, + "pass@2": 0.8980640886172121, + "pass@4": 0.9245809564374623, + "pass@8": 0.9405203926008627, + "pass@16": 0.9525469091473052, + "pass@32": 0.963199459193292, + "pass@64": 0.9716362530225532, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7924715909090909, + "pass@2": 0.861036670842281, + "pass@4": 0.9029049320539471, + "pass@8": 0.9281268028368013, + "pass@16": 0.9426543876503707, + "pass@32": 0.9525476884583312, + "pass@64": 0.9611302965599587, + "pass@128": 0.9696969696969697 + }, + "16": { + "pass@1": 0.6614583333333334, + "pass@2": 0.757938160374184, + "pass@4": 0.8144365875205767, + "pass@8": 0.8475187224277707, + "pass@16": 0.8701097260916296, + "pass@32": 0.8890719227083439, + "pass@64": 0.9063072562149335, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5830260093167702, + "pass@2": 0.6846121986110436, + "pass@4": 0.7533166365656156, + "pass@8": 0.8007323980276663, + "pass@16": 0.8367999826994603, + "pass@32": 0.8654849657428961, + "pass@64": 0.8891736360856827, + "pass@128": 0.906832298136646 + }, + "18": { + "pass@1": 0.5104779411764706, + "pass@2": 0.615626447429365, + "pass@4": 0.6993681317408854, + "pass@8": 0.7636960252154447, + "pass@16": 0.8126314154686012, + "pass@32": 0.8493046683312366, + "pass@64": 0.8760405951252511, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.33535621279761907, + "pass@2": 0.450107066694788, + "pass@4": 0.5589451405069903, + "pass@8": 0.6479033713249467, + "pass@16": 0.7114656294943115, + "pass@32": 0.7547122095604747, + "pass@64": 0.7871870069795173, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.30761133982035926, + "pass@2": 0.4103107760856239, + "pass@4": 0.5020851449519407, + "pass@8": 0.577641649195555, + "pass@16": 0.640643393523646, + "pass@32": 0.6937003584791263, + "pass@64": 0.7400380698580312, + "pass@128": 0.7844311377245509 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 651496, + "answer_accuracy": 0.8131989934494328, + "avg_response_len": 304.374299758348, + "resp_tokens_sum": 243850079, + "pass_at_k": { + "pass@1": 0.8131989934494328, + "pass@2": 0.8516683416824594, + "pass@4": 0.8803690842367309, + "pass@8": 0.9015596188301824, + "pass@16": 0.9172829494592375, + "pass@32": 0.9293968706923023, + "pass@64": 0.9392539645433563, + "pass@128": 0.9475954625339511 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9517957089552239, + "pass@2": 0.9643014014572805, + "pass@4": 0.9723098293683435, + "pass@8": 0.9778178687535791, + "pass@16": 0.9818233351333638, + "pass@32": 0.9843700251647998, + "pass@64": 0.9862015157581675, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998861151603499, + "pass@2": 0.99999784784555, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979669225146199, + "pass@2": 0.9986840666298292, + "pass@4": 0.9994179578978066, + "pass@8": 0.9998914828566827, + "pass@16": 0.999996971910131, + "pass@32": 0.9999999992108352, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915670955882353, + "pass@2": 0.9952162459471977, + "pass@4": 0.996770337898939, + "pass@8": 0.9975063173915031, + "pass@16": 0.9980350056586529, + "pass@32": 0.9987689956690733, + "pass@64": 0.9996410375173692, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964264589905363, + "pass@2": 0.9985430276459921, + "pass@4": 0.9995857823568582, + "pass@8": 0.9999535526784058, + "pass@16": 0.9999995379907698, + "pass@32": 0.9999999999901541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972804588607594, + "pass@2": 0.9992676511262834, + "pass@4": 0.9999203209764919, + "pass@8": 0.9999990565168133, + "pass@16": 0.9999999999228011, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9907563813813813, + "pass@2": 0.99560153164976, + "pass@4": 0.998587598565945, + "pass@8": 0.9998129486244137, + "pass@16": 0.999996824080759, + "pass@32": 0.9999999996818268, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9906300241157556, + "pass@2": 0.9950913993467858, + "pass@4": 0.9970629562582811, + "pass@8": 0.9986055619182425, + "pass@16": 0.99964486782007, + "pass@32": 0.9999770906276318, + "pass@64": 0.9999999831714547, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9624007936507937, + "pass@2": 0.9748172103487065, + "pass@4": 0.981273297087864, + "pass@8": 0.9846499737188555, + "pass@16": 0.9862178333308695, + "pass@32": 0.9878187753676408, + "pass@64": 0.9897012873390828, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9143969801223242, + "pass@2": 0.9317818572563751, + "pass@4": 0.941430099907236, + "pass@8": 0.9493256684380422, + "pass@16": 0.9570131901299108, + "pass@32": 0.9641514897698327, + "pass@64": 0.9703591098246457, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8329957561728395, + "pass@2": 0.8691643579274818, + "pass@4": 0.8912119784216789, + "pass@8": 0.9091026647516887, + "pass@16": 0.9252978369245227, + "pass@32": 0.9383556719133557, + "pass@64": 0.9473490414410928, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8191533753709199, + "pass@2": 0.8682212201219658, + "pass@4": 0.8979430763847394, + "pass@8": 0.9180236634394026, + "pass@16": 0.9314134024114261, + "pass@32": 0.9406145618745281, + "pass@64": 0.9479964540280973, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.7535927854938271, + "pass@2": 0.8146192554923689, + "pass@4": 0.8533767113022905, + "pass@8": 0.878865995821382, + "pass@16": 0.8964124781429048, + "pass@32": 0.910541254887923, + "pass@64": 0.922806998472301, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.6453032817109144, + "pass@2": 0.7468509017954611, + "pass@4": 0.8109872092868917, + "pass@8": 0.8503621602906531, + "pass@16": 0.8763865513376642, + "pass@32": 0.8958750335367089, + "pass@64": 0.9123815407896373, + "pass@128": 0.9262536873156342 + }, + "17": { + "pass@1": 0.5255553463855421, + "pass@2": 0.6341747402997818, + "pass@4": 0.7174640033700606, + "pass@8": 0.7752305050970983, + "pass@16": 0.8126617684952755, + "pass@32": 0.8371897052025431, + "pass@64": 0.8560389765187822, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4767389112903226, + "pass@2": 0.5847179006858011, + "pass@4": 0.66608676022352, + "pass@8": 0.7243191147240607, + "pass@16": 0.7663746277687996, + "pass@32": 0.8006317366757167, + "pass@64": 0.829606389878158, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.30501793806646527, + "pass@2": 0.40979077955134763, + "pass@4": 0.5044869689627164, + "pass@8": 0.5830466910339542, + "pass@16": 0.6470709277754211, + "pass@32": 0.7023998688940181, + "pass@64": 0.7524830182120951, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.2891798048048048, + "pass@2": 0.3927472009410985, + "pass@4": 0.49233214992270086, + "pass@8": 0.5773029346421416, + "pass@16": 0.6449435379481746, + "pass@32": 0.6935367729692132, + "pass@64": 0.7276704804486359, + "pass@128": 0.7567567567567568 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..acf0c176fa0ab29e2b705eb424b6b139498bc71a --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9fea81e38268b0cf4fddf50b43439227beea5c926207e592d283cd10c6270c2 +size 1790456324 diff --git a/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json b/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7be54959205e1d861d6c1aa715862c470b93df03 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op17-20", + "id": { + "answer_accuracy": 0.9890277777777777, + "avg_response_len": 199.14139930555555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9890277777777777, + "pass@2": 0.9928172162073456, + "pass@4": 0.9948839220097478, + "pass@8": 0.9963080761191235, + "pass@16": 0.9973152803179801, + "pass@32": 0.9980268388493161, + "pass@64": 0.99847883565242, + "pass@128": 0.9987777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391875, + "pass@2": 0.9553868110236222, + "pass@4": 0.9664699572553431, + "pass@8": 0.9749212520185588, + "pass@16": 0.9816938886763031, + "pass@32": 0.9868280196969069, + "pass@64": 0.989992463600238, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999991387795276, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99996875, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99815625, + "pass@2": 0.9986737204724409, + "pass@4": 0.9989233598612672, + "pass@8": 0.9990519583151254, + "pass@16": 0.9991249279427927, + "pass@32": 0.9992499999993945, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.992640625, + "pass@2": 0.9958212352362207, + "pass@4": 0.997663693007124, + "pass@8": 0.9989601583261803, + "pass@16": 0.9996727318798748, + "pass@32": 0.9999476997271504, + "pass@64": 0.9999993318674036, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955546875, + "pass@2": 0.9968681102362207, + "pass@4": 0.9975768131796027, + "pass@8": 0.9981352577610507, + "pass@16": 0.9986862301248822, + "pass@32": 0.9992675062194463, + "pass@64": 0.9998183933036234, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9961640625, + "pass@2": 0.9984712106299214, + "pass@4": 0.9994211929133856, + "pass@8": 0.9998762765811555, + "pass@16": 0.9999913086119966, + "pass@32": 0.9999999700015125, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9908046875, + "pass@2": 0.995634965551181, + "pass@4": 0.9976798744844393, + "pass@8": 0.998685442665731, + "pass@16": 0.998977556450126, + "pass@32": 0.9989998945869978, + "pass@64": 0.9989999999999175, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9888203125, + "pass@2": 0.9944998769685044, + "pass@4": 0.9962204074803149, + "pass@8": 0.9971423394043094, + "pass@16": 0.9976908791758443, + "pass@32": 0.997948459412436, + "pass@64": 0.9979993321006008, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9391875, + "2": 0.999953125, + "3": 0.99996875, + "4": 0.99815625, + "5": 0.992640625, + "6": 0.9955546875, + "7": 0.9961640625, + "8": 0.9908046875, + "9": 0.9888203125 + }, + "per_op_avg_response_len": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375474, + "answer_accuracy": 0.9906756585612969, + "avg_response_len": 189.4462069402229, + "resp_tokens_sum": 71801628, + "pass_at_k": { + "pass@1": 0.9906756585612969, + "pass@2": 0.994166370825988, + "pass@4": 0.9958945190108589, + "pass@8": 0.9970284643001964, + "pass@16": 0.997750665029614, + "pass@32": 0.9982530045544682, + "pass@64": 0.9987353591726881, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9463956489675516, + "pass@2": 0.9599948029173347, + "pass@4": 0.9696439955514411, + "pass@8": 0.9772987780870966, + "pass@16": 0.9829576313879089, + "pass@32": 0.9869532680938505, + "pass@64": 0.9904289041602642, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962890625, + "pass@2": 0.9968965305118112, + "pass@4": 0.9969725551884139, + "pass@8": 0.9970703124996394, + "pass@16": 0.997265625, + "pass@32": 0.99765625, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941220238095239, + "pass@2": 0.9976346706661666, + "pass@4": 0.9991279667422525, + "pass@8": 0.999810090374339, + "pass@16": 0.9999911414698637, + "pass@32": 0.9999999920410901, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9975673652694611, + "pass@2": 0.9990827891461174, + "pass@4": 0.9997700759897528, + "pass@8": 0.9999845197580833, + "pass@16": 0.9999999528630369, + "pass@32": 0.9999999999999486, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960012943786982, + "pass@2": 0.9984763022410661, + "pass@4": 0.9996465231971745, + "pass@8": 0.9999815406455984, + "pass@16": 0.9999999678464966, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963441506410257, + "pass@2": 0.9996159209065214, + "pass@4": 0.9999852656639074, + "pass@8": 0.9999999592877328, + "pass@16": 0.9999999999999242, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9913292253521127, + "pass@2": 0.9970926444493733, + "pass@4": 0.9988132281880259, + "pass@8": 0.9997344814601936, + "pass@16": 0.9999856314389629, + "pass@32": 0.9999999749550461, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386021, + "answer_accuracy": 0.9881353415792923, + "avg_response_len": 197.20948609518348, + "resp_tokens_sum": 77041069, + "pass_at_k": { + "pass@1": 0.9881353415792923, + "pass@2": 0.991507118089597, + "pass@4": 0.9934450351819555, + "pass@8": 0.9948278390202879, + "pass@16": 0.995978093148729, + "pass@32": 0.9968872544058474, + "pass@64": 0.9974407893814518, + "pass@128": 0.9977064220183486 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924079754601227, + "pass@2": 0.9425480653108548, + "pass@4": 0.9558939233439372, + "pass@8": 0.9660136894297928, + "pass@16": 0.9745849213182081, + "pass@32": 0.9815053815983932, + "pass@64": 0.9856196459168264, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920969202898551, + "pass@2": 0.9947952327969871, + "pass@4": 0.9963463162213417, + "pass@8": 0.9977460600531718, + "pass@16": 0.9990916148451592, + "pass@32": 0.9998484564379228, + "pass@64": 0.999998063383779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966728617196489, + "pass@4": 0.9972907742728435, + "pass@8": 0.997639465969732, + "pass@16": 0.9980869583411133, + "pass@32": 0.9988007408816018, + "pass@64": 0.9996502944295288, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9983024498202175, + "pass@4": 0.9990612040546953, + "pass@8": 0.9997078494159238, + "pass@16": 0.999975444312545, + "pass@32": 0.9999999133144741, + "pass@64": 0.9999999999999895, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918573943661971, + "pass@2": 0.9956231978485083, + "pass@4": 0.9969242494336096, + "pass@8": 0.9971736180162805, + "pass@16": 0.9971830890857465, + "pass@32": 0.9971830985915483, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9910179640718563, + "pass@2": 0.9937379178650573, + "pass@4": 0.9939987547402381, + "pass@8": 0.9940119380249567, + "pass@16": 0.9940119760478334, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 377865, + "answer_accuracy": 0.9883060972547707, + "avg_response_len": 210.72615448192167, + "resp_tokens_sum": 80568195, + "pass_at_k": { + "pass@1": 0.9883060972547707, + "pass@2": 0.9928184122668052, + "pass@4": 0.9953523200272051, + "pass@8": 0.9971064069599305, + "pass@16": 0.9982499710141318, + "pass@32": 0.9989670246774082, + "pass@64": 0.9992851801704917, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465951492537313, + "pass@2": 0.9632176225173348, + "pass@4": 0.9735499216515845, + "pass@8": 0.9811835925161799, + "pass@16": 0.9873330366748251, + "pass@32": 0.9918809176478053, + "pass@64": 0.993806150809024, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981953581871345, + "pass@2": 0.9990279792328591, + "pass@4": 0.9996846265927285, + "pass@8": 0.9999691763603531, + "pass@16": 0.9999997893064115, + "pass@32": 0.9999999999982297, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9918198529411765, + "pass@2": 0.9951822313571099, + "pass@4": 0.9976438364322108, + "pass@8": 0.9994046739409391, + "pass@16": 0.9999673976331993, + "pass@32": 0.9999999551856581, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.993764787066246, + "pass@2": 0.9947496212027123, + "pass@4": 0.9955808440940151, + "pass@8": 0.9967326641590364, + "pass@16": 0.9979618214750134, + "pass@32": 0.9990096140434843, + "pass@64": 0.9998121152924854, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9986505469450812, + "pass@4": 0.9995743400943553, + "pass@8": 0.9999481010918784, + "pass@16": 0.9999994169232284, + "pass@32": 0.9999999999832521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9844923048048048, + "pass@2": 0.9919176065829609, + "pass@4": 0.9963254145596664, + "pass@8": 0.9990655284449819, + "pass@16": 0.9999326120862155, + "pass@32": 0.9999996834444386, + "pass@64": 0.999999999999752, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9835962620578779, + "pass@2": 0.9923585968554577, + "pass@4": 0.9956467118660005, + "pass@8": 0.9975453735871543, + "pass@16": 0.9990224437782512, + "pass@32": 0.9998343032263491, + "pass@64": 0.9999978524135079, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6918046875, + "avg_response_len": 372.07711328125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6918046875, + "pass@2": 0.7649430733267603, + "pass@4": 0.8173947985658003, + "pass@8": 0.8545082829889343, + "pass@16": 0.8807741388344542, + "pass@32": 0.9002820731595781, + "pass@64": 0.9154065233838675, + "pass@128": 0.9274 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.948625, + "pass@2": 0.9650340797244101, + "pass@4": 0.974049004686914, + "pass@8": 0.9799215853539738, + "pass@16": 0.9840942258978155, + "pass@32": 0.9873766462931601, + "pass@64": 0.9897657015488682, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.8919921875, + "pass@2": 0.9173727854330723, + "pass@4": 0.9306753659542553, + "pass@8": 0.9402165770631085, + "pass@16": 0.9484162200957359, + "pass@32": 0.9556793185220008, + "pass@64": 0.9622626730700722, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.8227109375, + "pass@2": 0.8682181348425212, + "pass@4": 0.8953016328271474, + "pass@8": 0.9137342715759691, + "pass@16": 0.9279086259612404, + "pass@32": 0.9409428563904914, + "pass@64": 0.9531583365904523, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.8139765625, + "pass@2": 0.8710821850393708, + "pass@4": 0.9045889305399336, + "pass@8": 0.9250908670684431, + "pass@16": 0.9386299476507689, + "pass@32": 0.9487427582732499, + "pass@64": 0.957440936100577, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.7839609375, + "pass@2": 0.8494299950787403, + "pass@4": 0.8907248334270719, + "pass@8": 0.9178499015041497, + "pass@16": 0.9354141739723325, + "pass@32": 0.9468501950584634, + "pass@64": 0.9547623055339345, + "pass@128": 0.962 + }, + "16": { + "pass@1": 0.66109375, + "pass@2": 0.7585896899606303, + "pass@4": 0.8241256755718038, + "pass@8": 0.8659143995718789, + "pass@16": 0.8935886268502504, + "pass@32": 0.9145088226574314, + "pass@64": 0.9303470680153623, + "pass@128": 0.941 + }, + "17": { + "pass@1": 0.605640625, + "pass@2": 0.7034212598425199, + "pass@4": 0.7721566392950879, + "pass@8": 0.8196972052982548, + "pass@16": 0.8530516738610301, + "pass@32": 0.876275545986001, + "pass@64": 0.8930550155935169, + "pass@128": 0.906 + }, + "18": { + "pass@1": 0.56478125, + "pass@2": 0.6675775098425195, + "pass@4": 0.7450145286839147, + "pass@8": 0.7996635802319876, + "pass@16": 0.8369695037285299, + "pass@32": 0.8625344989141525, + "pass@64": 0.8809200646871671, + "pass@128": 0.896 + }, + "19": { + "pass@1": 0.413765625, + "pass@2": 0.5293097933070865, + "pass@4": 0.6272639149793771, + "pass@8": 0.7019809692420499, + "pass@16": 0.7557623698652861, + "pass@32": 0.7956271976826763, + "pass@64": 0.8254726377576067, + "pass@128": 0.848 + }, + "20": { + "pass@1": 0.4115, + "pass@2": 0.5193953001968503, + "pass@4": 0.6100474596925382, + "pass@8": 0.6810134729795795, + "pass@16": 0.7339060204614524, + "pass@32": 0.7742828918181295, + "pass@64": 0.8068804949410628, + "pass@128": 0.833 + } + }, + "per_op_accuracy": { + "11": 0.948625, + "12": 0.8919921875, + "13": 0.8227109375, + "14": 0.8139765625, + "15": 0.7839609375, + "16": 0.66109375, + "17": 0.605640625, + "18": 0.56478125, + "19": 0.413765625, + "20": 0.4115 + }, + "per_op_avg_response_len": { + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 301159, + "answer_accuracy": 0.6991990156017831, + "avg_response_len": 353.8343819650817, + "resp_tokens_sum": 152403545, + "pass_at_k": { + "pass@1": 0.6991990156017831, + "pass@2": 0.7698041221583997, + "pass@4": 0.8194136314684324, + "pass@8": 0.8540800124080026, + "pass@16": 0.8791774558830416, + "pass@32": 0.8984448726143027, + "pass@64": 0.9136238088950139, + "pass@128": 0.9260029717682021 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9559055815172538, + "pass@4": 0.9662483132420059, + "pass@8": 0.9743145176794873, + "pass@16": 0.9814528957415205, + "pass@32": 0.9870929428344121, + "pass@64": 0.9904799574420998, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8917284604519774, + "pass@2": 0.9156980542951199, + "pass@4": 0.9281065293215466, + "pass@8": 0.935889004485147, + "pass@16": 0.9418117371185404, + "pass@32": 0.947600022930526, + "pass@64": 0.9542037056880797, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8402233542319749, + "pass@2": 0.8852274269987415, + "pass@4": 0.9120427055748123, + "pass@8": 0.9300158215455117, + "pass@16": 0.9440746211517979, + "pass@32": 0.9571960077579881, + "pass@64": 0.9683369401551648, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8298773364485982, + "pass@2": 0.8854661092305048, + "pass@4": 0.9165421769708681, + "pass@8": 0.9342494936091128, + "pass@16": 0.9469401578089754, + "pass@32": 0.9577873713575069, + "pass@64": 0.9667747235355996, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7909365968208093, + "pass@2": 0.8548815057348322, + "pass@4": 0.8965234123552469, + "pass@8": 0.9232631518428708, + "pass@16": 0.93956332213618, + "pass@32": 0.9494509870773286, + "pass@64": 0.9560635736921728, + "pass@128": 0.9624277456647399 + }, + "16": { + "pass@1": 0.6512096774193549, + "pass@2": 0.7479818865887731, + "pass@4": 0.8152397825271837, + "pass@8": 0.8589674413513186, + "pass@16": 0.8885187025219686, + "pass@32": 0.9122664817583825, + "pass@64": 0.9310470742858282, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6124909682080925, + "pass@2": 0.7055230474261528, + "pass@4": 0.768419161297034, + "pass@8": 0.8132989938840093, + "pass@16": 0.8486630253769624, + "pass@32": 0.8756942867659188, + "pass@64": 0.8953656362346946, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5464732142857143, + "pass@2": 0.6480627812148483, + "pass@4": 0.7236771966896989, + "pass@8": 0.777945603229985, + "pass@16": 0.8163166798894307, + "pass@32": 0.8431802277020857, + "pass@64": 0.8622731277447071, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.4315643768768769, + "pass@2": 0.5412484334728428, + "pass@4": 0.6303853890448375, + "pass@8": 0.6978895463021549, + "pass@16": 0.7471806522306299, + "pass@32": 0.7842191914351475, + "pass@64": 0.8144645683656517, + "pass@128": 0.8438438438438438 + }, + "20": { + "pass@1": 0.4473066816816817, + "pass@2": 0.550585157007401, + "pass@4": 0.6322523291007539, + "pass@8": 0.6917831399797562, + "pass@16": 0.7350186424328486, + "pass@32": 0.7688185050754536, + "pass@64": 0.7971501932040521, + "pass@128": 0.8228228228228228 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 300394, + "answer_accuracy": 0.6978376821290514, + "avg_response_len": 367.61922715953017, + "resp_tokens_sum": 158246843, + "pass_at_k": { + "pass@1": 0.6978376821290514, + "pass@2": 0.7712399262703721, + "pass@4": 0.8234056467183366, + "pass@8": 0.8598974993623416, + "pass@16": 0.8856925425332236, + "pass@32": 0.9053746000361343, + "pass@64": 0.9216224010043141, + "pass@128": 0.9351769253642581 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9533838478915663, + "pass@2": 0.9686280802343229, + "pass@4": 0.9774122846842942, + "pass@8": 0.9826945678160793, + "pass@16": 0.9851505016991625, + "pass@32": 0.9862401453810479, + "pass@64": 0.9872047186881181, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.885384012539185, + "pass@2": 0.9101669525831221, + "pass@4": 0.9239799511738148, + "pass@8": 0.9333532233276649, + "pass@16": 0.9404962983721141, + "pass@32": 0.947095869663446, + "pass@64": 0.9545776664683873, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8328956582633054, + "pass@2": 0.8768782119146881, + "pass@4": 0.9001525788793208, + "pass@8": 0.9144132901889223, + "pass@16": 0.9252804791130302, + "pass@32": 0.9358350221164605, + "pass@64": 0.9474943865376289, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.8158351608187134, + "pass@2": 0.8764760182115391, + "pass@4": 0.9120771337245117, + "pass@8": 0.9326000038402523, + "pass@16": 0.9449230559731131, + "pass@32": 0.9539728291687525, + "pass@64": 0.9628130945239121, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.7901515151515152, + "pass@2": 0.8613483506323071, + "pass@4": 0.9039200494824501, + "pass@8": 0.9292158194336342, + "pass@16": 0.9452248133475442, + "pass@32": 0.9574867632372372, + "pass@64": 0.9678628579175225, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6649528133903134, + "pass@2": 0.7624254800681963, + "pass@4": 0.8277979675617472, + "pass@8": 0.8686706781122179, + "pass@16": 0.8961598925297205, + "pass@32": 0.9179229188572494, + "pass@64": 0.9337860632473033, + "pass@128": 0.9430199430199431 + }, + "17": { + "pass@1": 0.6393390916149069, + "pass@2": 0.7354016634469608, + "pass@4": 0.8004865514675568, + "pass@8": 0.8453401336925654, + "pass@16": 0.8772971858982275, + "pass@32": 0.9004220002674332, + "pass@64": 0.9174945886997983, + "pass@128": 0.9316770186335404 + }, + "18": { + "pass@1": 0.5892693014705882, + "pass@2": 0.6895900880037054, + "pass@4": 0.7655650637420315, + "pass@8": 0.8203016454124229, + "pass@16": 0.859618198955504, + "pass@32": 0.8878937646484227, + "pass@64": 0.9097340634600377, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.4204799107142857, + "pass@2": 0.5389387361736032, + "pass@4": 0.6410350558858714, + "pass@8": 0.7190375968217035, + "pass@16": 0.7736645475875377, + "pass@32": 0.8123192037757934, + "pass@64": 0.8394299819521862, + "pass@128": 0.8571428571428571 + }, + "20": { + "pass@1": 0.3888005239520958, + "pass@2": 0.4932697351360271, + "pass@4": 0.581708334175294, + "pass@8": 0.6533021743606491, + "pass@16": 0.7090402348266315, + "pass@32": 0.754332532451074, + "pass@64": 0.7954232145576876, + "pass@128": 0.8293413173652695 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 283957, + "answer_accuracy": 0.6779994078545232, + "avg_response_len": 395.4202251107885, + "resp_tokens_sum": 165608317, + "pass_at_k": { + "pass@1": 0.6779994078545232, + "pass@2": 0.7534718796324861, + "pass@4": 0.8091405641360095, + "pass@8": 0.8494096264611646, + "pass@16": 0.8773610112343615, + "pass@32": 0.8969373335351778, + "pass@64": 0.9108511559686301, + "pass@128": 0.9208435207823961 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9548611111111112, + "pass@2": 0.9714758311461071, + "pass@4": 0.9792459415787311, + "pass@8": 0.9832824257084963, + "pass@16": 0.9859409115458316, + "pass@32": 0.9888924101781106, + "pass@64": 0.9916644760868305, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8987241972477065, + "pass@2": 0.9262153404849626, + "pass@4": 0.9399879209479548, + "pass@8": 0.9515969150881995, + "pass@16": 0.9632921895262051, + "pass@32": 0.9727991681405372, + "pass@64": 0.978484053984944, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7942467206790124, + "pass@2": 0.8419292098036356, + "pass@4": 0.87347388607674, + "pass@8": 0.896955802177486, + "pass@16": 0.9148879344767462, + "pass@32": 0.9305686019139384, + "pass@64": 0.9444548045895704, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7969445474777448, + "pass@2": 0.8519073167597371, + "pass@4": 0.8856038931706228, + "pass@8": 0.9087465231648425, + "pass@16": 0.9243278096477272, + "pass@32": 0.9348198946343538, + "pass@64": 0.9430984318055536, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.7702064043209876, + "pass@2": 0.8314692542772435, + "pass@4": 0.8710929519226761, + "pass@8": 0.9004926868932659, + "pass@16": 0.9209909756435962, + "pass@32": 0.9332392643870973, + "pass@64": 0.9400295244557415, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.6661366150442478, + "pass@2": 0.7643184708382698, + "pass@4": 0.8284491338472064, + "pass@8": 0.8694132293085024, + "pass@16": 0.8955625569041571, + "pass@32": 0.9130243917210557, + "pass@64": 0.926146214710774, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.5658179593373494, + "pass@2": 0.6702137042263541, + "pass@4": 0.7485751202221707, + "pass@8": 0.8014946697752476, + "pass@16": 0.8341101603053381, + "pass@32": 0.8534621646954216, + "pass@64": 0.8669435177559548, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.55859375, + "pass@2": 0.6654674403098803, + "pass@4": 0.7465657682910599, + "pass@8": 0.801548579552477, + "pass@16": 0.8354467036205091, + "pass@32": 0.8565727072192222, + "pass@64": 0.8703706077422788, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.3890436178247734, + "pass@2": 0.5075246211670672, + "pass@4": 0.6101444158604039, + "pass@8": 0.6887828332064626, + "pass@16": 0.7462233374111003, + "pass@32": 0.7901599832812849, + "pass@64": 0.8223790409544078, + "pass@128": 0.8429003021148036 + }, + "20": { + "pass@1": 0.39846096096096095, + "pass@2": 0.5144094635974552, + "pass@4": 0.6162668182805975, + "pass@8": 0.6980383217113035, + "pass@16": 0.7577338561538128, + "pass@32": 0.799757548916951, + "pass@64": 0.828102483308844, + "pass@128": 0.8468468468468469 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.832594572368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.832594572368421, + "pass@2": 0.8728834567964937, + "pass@4": 0.9014685938813386, + "pass@8": 0.9216766060506145, + "pass@16": 0.9359778374318961, + "pass@32": 0.9465822253284079, + "pass@64": 0.954756566037391, + "pass@128": 0.9612105263157895 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391875, + "pass@2": 0.9553868110236222, + "pass@4": 0.9664699572553431, + "pass@8": 0.9749212520185588, + "pass@16": 0.9816938886763031, + "pass@32": 0.9868280196969069, + "pass@64": 0.989992463600238, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999991387795276, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99996875, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99815625, + "pass@2": 0.9986737204724409, + "pass@4": 0.9989233598612672, + "pass@8": 0.9990519583151254, + "pass@16": 0.9991249279427927, + "pass@32": 0.9992499999993945, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.992640625, + "pass@2": 0.9958212352362207, + "pass@4": 0.997663693007124, + "pass@8": 0.9989601583261803, + "pass@16": 0.9996727318798748, + "pass@32": 0.9999476997271504, + "pass@64": 0.9999993318674036, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955546875, + "pass@2": 0.9968681102362207, + "pass@4": 0.9975768131796027, + "pass@8": 0.9981352577610507, + "pass@16": 0.9986862301248822, + "pass@32": 0.9992675062194463, + "pass@64": 0.9998183933036234, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9961640625, + "pass@2": 0.9984712106299214, + "pass@4": 0.9994211929133856, + "pass@8": 0.9998762765811555, + "pass@16": 0.9999913086119966, + "pass@32": 0.9999999700015125, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9908046875, + "pass@2": 0.995634965551181, + "pass@4": 0.9976798744844393, + "pass@8": 0.998685442665731, + "pass@16": 0.998977556450126, + "pass@32": 0.9989998945869978, + "pass@64": 0.9989999999999175, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9888203125, + "pass@2": 0.9944998769685044, + "pass@4": 0.9962204074803149, + "pass@8": 0.9971423394043094, + "pass@16": 0.9976908791758443, + "pass@32": 0.997948459412436, + "pass@64": 0.9979993321006008, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.948625, + "pass@2": 0.9650340797244101, + "pass@4": 0.974049004686914, + "pass@8": 0.9799215853539738, + "pass@16": 0.9840942258978155, + "pass@32": 0.9873766462931601, + "pass@64": 0.9897657015488682, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.8919921875, + "pass@2": 0.9173727854330723, + "pass@4": 0.9306753659542553, + "pass@8": 0.9402165770631085, + "pass@16": 0.9484162200957359, + "pass@32": 0.9556793185220008, + "pass@64": 0.9622626730700722, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.8227109375, + "pass@2": 0.8682181348425212, + "pass@4": 0.8953016328271474, + "pass@8": 0.9137342715759691, + "pass@16": 0.9279086259612404, + "pass@32": 0.9409428563904914, + "pass@64": 0.9531583365904523, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.8139765625, + "pass@2": 0.8710821850393708, + "pass@4": 0.9045889305399336, + "pass@8": 0.9250908670684431, + "pass@16": 0.9386299476507689, + "pass@32": 0.9487427582732499, + "pass@64": 0.957440936100577, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.7839609375, + "pass@2": 0.8494299950787403, + "pass@4": 0.8907248334270719, + "pass@8": 0.9178499015041497, + "pass@16": 0.9354141739723325, + "pass@32": 0.9468501950584634, + "pass@64": 0.9547623055339345, + "pass@128": 0.962 + }, + "16": { + "pass@1": 0.66109375, + "pass@2": 0.7585896899606303, + "pass@4": 0.8241256755718038, + "pass@8": 0.8659143995718789, + "pass@16": 0.8935886268502504, + "pass@32": 0.9145088226574314, + "pass@64": 0.9303470680153623, + "pass@128": 0.941 + }, + "17": { + "pass@1": 0.605640625, + "pass@2": 0.7034212598425199, + "pass@4": 0.7721566392950879, + "pass@8": 0.8196972052982548, + "pass@16": 0.8530516738610301, + "pass@32": 0.876275545986001, + "pass@64": 0.8930550155935169, + "pass@128": 0.906 + }, + "18": { + "pass@1": 0.56478125, + "pass@2": 0.6675775098425195, + "pass@4": 0.7450145286839147, + "pass@8": 0.7996635802319876, + "pass@16": 0.8369695037285299, + "pass@32": 0.8625344989141525, + "pass@64": 0.8809200646871671, + "pass@128": 0.896 + }, + "19": { + "pass@1": 0.413765625, + "pass@2": 0.5293097933070865, + "pass@4": 0.6272639149793771, + "pass@8": 0.7019809692420499, + "pass@16": 0.7557623698652861, + "pass@32": 0.7956271976826763, + "pass@64": 0.8254726377576067, + "pass@128": 0.848 + }, + "20": { + "pass@1": 0.4115, + "pass@2": 0.5193953001968503, + "pass@4": 0.6100474596925382, + "pass@8": 0.6810134729795795, + "pass@16": 0.7339060204614524, + "pass@32": 0.7742828918181295, + "pass@64": 0.8068804949410628, + "pass@128": 0.833 + } + }, + "per_op_accuracy": { + "10": 0.9391875, + "2": 0.999953125, + "3": 0.99996875, + "4": 0.99815625, + "5": 0.992640625, + "6": 0.9955546875, + "7": 0.9961640625, + "8": 0.9908046875, + "9": 0.9888203125, + "11": 0.948625, + "12": 0.8919921875, + "13": 0.8227109375, + "14": 0.8139765625, + "15": 0.7839609375, + "16": 0.66109375, + "17": 0.605640625, + "18": 0.56478125, + "19": 0.413765625, + "20": 0.4115 + }, + "per_op_avg_response_len": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375, + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375, + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 676633, + "answer_accuracy": 0.8356299893297502, + "avg_response_len": 276.8894900509801, + "resp_tokens_sum": 224205173, + "pass_at_k": { + "pass@1": 0.8356299893297502, + "pass@2": 0.8748209761426936, + "pass@4": 0.9020187386472343, + "pass@8": 0.9209896497859279, + "pass@16": 0.9346778150804824, + "pass@32": 0.9451618942195561, + "pass@64": 0.9534618266585658, + "pass@128": 0.9603224786595005 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9463956489675516, + "pass@2": 0.9599948029173347, + "pass@4": 0.9696439955514411, + "pass@8": 0.9772987780870966, + "pass@16": 0.9829576313879089, + "pass@32": 0.9869532680938505, + "pass@64": 0.9904289041602642, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962890625, + "pass@2": 0.9968965305118112, + "pass@4": 0.9969725551884139, + "pass@8": 0.9970703124996394, + "pass@16": 0.997265625, + "pass@32": 0.99765625, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941220238095239, + "pass@2": 0.9976346706661666, + "pass@4": 0.9991279667422525, + "pass@8": 0.999810090374339, + "pass@16": 0.9999911414698637, + "pass@32": 0.9999999920410901, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9975673652694611, + "pass@2": 0.9990827891461174, + "pass@4": 0.9997700759897528, + "pass@8": 0.9999845197580833, + "pass@16": 0.9999999528630369, + "pass@32": 0.9999999999999486, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960012943786982, + "pass@2": 0.9984763022410661, + "pass@4": 0.9996465231971745, + "pass@8": 0.9999815406455984, + "pass@16": 0.9999999678464966, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963441506410257, + "pass@2": 0.9996159209065214, + "pass@4": 0.9999852656639074, + "pass@8": 0.9999999592877328, + "pass@16": 0.9999999999999242, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9913292253521127, + "pass@2": 0.9970926444493733, + "pass@4": 0.9988132281880259, + "pass@8": 0.9997344814601936, + "pass@16": 0.9999856314389629, + "pass@32": 0.9999999749550461, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9559055815172538, + "pass@4": 0.9662483132420059, + "pass@8": 0.9743145176794873, + "pass@16": 0.9814528957415205, + "pass@32": 0.9870929428344121, + "pass@64": 0.9904799574420998, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8917284604519774, + "pass@2": 0.9156980542951199, + "pass@4": 0.9281065293215466, + "pass@8": 0.935889004485147, + "pass@16": 0.9418117371185404, + "pass@32": 0.947600022930526, + "pass@64": 0.9542037056880797, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8402233542319749, + "pass@2": 0.8852274269987415, + "pass@4": 0.9120427055748123, + "pass@8": 0.9300158215455117, + "pass@16": 0.9440746211517979, + "pass@32": 0.9571960077579881, + "pass@64": 0.9683369401551648, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8298773364485982, + "pass@2": 0.8854661092305048, + "pass@4": 0.9165421769708681, + "pass@8": 0.9342494936091128, + "pass@16": 0.9469401578089754, + "pass@32": 0.9577873713575069, + "pass@64": 0.9667747235355996, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7909365968208093, + "pass@2": 0.8548815057348322, + "pass@4": 0.8965234123552469, + "pass@8": 0.9232631518428708, + "pass@16": 0.93956332213618, + "pass@32": 0.9494509870773286, + "pass@64": 0.9560635736921728, + "pass@128": 0.9624277456647399 + }, + "16": { + "pass@1": 0.6512096774193549, + "pass@2": 0.7479818865887731, + "pass@4": 0.8152397825271837, + "pass@8": 0.8589674413513186, + "pass@16": 0.8885187025219686, + "pass@32": 0.9122664817583825, + "pass@64": 0.9310470742858282, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6124909682080925, + "pass@2": 0.7055230474261528, + "pass@4": 0.768419161297034, + "pass@8": 0.8132989938840093, + "pass@16": 0.8486630253769624, + "pass@32": 0.8756942867659188, + "pass@64": 0.8953656362346946, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5464732142857143, + "pass@2": 0.6480627812148483, + "pass@4": 0.7236771966896989, + "pass@8": 0.777945603229985, + "pass@16": 0.8163166798894307, + "pass@32": 0.8431802277020857, + "pass@64": 0.8622731277447071, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.4315643768768769, + "pass@2": 0.5412484334728428, + "pass@4": 0.6303853890448375, + "pass@8": 0.6978895463021549, + "pass@16": 0.7471806522306299, + "pass@32": 0.7842191914351475, + "pass@64": 0.8144645683656517, + "pass@128": 0.8438438438438438 + }, + "20": { + "pass@1": 0.4473066816816817, + "pass@2": 0.550585157007401, + "pass@4": 0.6322523291007539, + "pass@8": 0.6917831399797562, + "pass@16": 0.7350186424328486, + "pass@32": 0.7688185050754536, + "pass@64": 0.7971501932040521, + "pass@128": 0.8228228228228228 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 686415, + "answer_accuracy": 0.8359496784879189, + "avg_response_len": 286.5450993764614, + "resp_tokens_sum": 235287912, + "pass_at_k": { + "pass@1": 0.8359496784879189, + "pass@2": 0.8760342317157656, + "pass@4": 0.9043035755711732, + "pass@8": 0.9240919493445696, + "pass@16": 0.9381619892173285, + "pass@32": 0.9489126547729012, + "pass@64": 0.9576937527310582, + "pass@128": 0.9649259547934529 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924079754601227, + "pass@2": 0.9425480653108548, + "pass@4": 0.9558939233439372, + "pass@8": 0.9660136894297928, + "pass@16": 0.9745849213182081, + "pass@32": 0.9815053815983932, + "pass@64": 0.9856196459168264, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920969202898551, + "pass@2": 0.9947952327969871, + "pass@4": 0.9963463162213417, + "pass@8": 0.9977460600531718, + "pass@16": 0.9990916148451592, + "pass@32": 0.9998484564379228, + "pass@64": 0.999998063383779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966728617196489, + "pass@4": 0.9972907742728435, + "pass@8": 0.997639465969732, + "pass@16": 0.9980869583411133, + "pass@32": 0.9988007408816018, + "pass@64": 0.9996502944295288, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9983024498202175, + "pass@4": 0.9990612040546953, + "pass@8": 0.9997078494159238, + "pass@16": 0.999975444312545, + "pass@32": 0.9999999133144741, + "pass@64": 0.9999999999999895, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918573943661971, + "pass@2": 0.9956231978485083, + "pass@4": 0.9969242494336096, + "pass@8": 0.9971736180162805, + "pass@16": 0.9971830890857465, + "pass@32": 0.9971830985915483, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9910179640718563, + "pass@2": 0.9937379178650573, + "pass@4": 0.9939987547402381, + "pass@8": 0.9940119380249567, + "pass@16": 0.9940119760478334, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9533838478915663, + "pass@2": 0.9686280802343229, + "pass@4": 0.9774122846842942, + "pass@8": 0.9826945678160793, + "pass@16": 0.9851505016991625, + "pass@32": 0.9862401453810479, + "pass@64": 0.9872047186881181, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.885384012539185, + "pass@2": 0.9101669525831221, + "pass@4": 0.9239799511738148, + "pass@8": 0.9333532233276649, + "pass@16": 0.9404962983721141, + "pass@32": 0.947095869663446, + "pass@64": 0.9545776664683873, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8328956582633054, + "pass@2": 0.8768782119146881, + "pass@4": 0.9001525788793208, + "pass@8": 0.9144132901889223, + "pass@16": 0.9252804791130302, + "pass@32": 0.9358350221164605, + "pass@64": 0.9474943865376289, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.8158351608187134, + "pass@2": 0.8764760182115391, + "pass@4": 0.9120771337245117, + "pass@8": 0.9326000038402523, + "pass@16": 0.9449230559731131, + "pass@32": 0.9539728291687525, + "pass@64": 0.9628130945239121, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.7901515151515152, + "pass@2": 0.8613483506323071, + "pass@4": 0.9039200494824501, + "pass@8": 0.9292158194336342, + "pass@16": 0.9452248133475442, + "pass@32": 0.9574867632372372, + "pass@64": 0.9678628579175225, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6649528133903134, + "pass@2": 0.7624254800681963, + "pass@4": 0.8277979675617472, + "pass@8": 0.8686706781122179, + "pass@16": 0.8961598925297205, + "pass@32": 0.9179229188572494, + "pass@64": 0.9337860632473033, + "pass@128": 0.9430199430199431 + }, + "17": { + "pass@1": 0.6393390916149069, + "pass@2": 0.7354016634469608, + "pass@4": 0.8004865514675568, + "pass@8": 0.8453401336925654, + "pass@16": 0.8772971858982275, + "pass@32": 0.9004220002674332, + "pass@64": 0.9174945886997983, + "pass@128": 0.9316770186335404 + }, + "18": { + "pass@1": 0.5892693014705882, + "pass@2": 0.6895900880037054, + "pass@4": 0.7655650637420315, + "pass@8": 0.8203016454124229, + "pass@16": 0.859618198955504, + "pass@32": 0.8878937646484227, + "pass@64": 0.9097340634600377, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.4204799107142857, + "pass@2": 0.5389387361736032, + "pass@4": 0.6410350558858714, + "pass@8": 0.7190375968217035, + "pass@16": 0.7736645475875377, + "pass@32": 0.8123192037757934, + "pass@64": 0.8394299819521862, + "pass@128": 0.8571428571428571 + }, + "20": { + "pass@1": 0.3888005239520958, + "pass@2": 0.4932697351360271, + "pass@4": 0.581708334175294, + "pass@8": 0.6533021743606491, + "pass@16": 0.7090402348266315, + "pass@32": 0.754332532451074, + "pass@64": 0.7954232145576876, + "pass@128": 0.8293413173652695 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 661822, + "answer_accuracy": 0.8260879333759387, + "avg_response_len": 307.27815945039146, + "resp_tokens_sum": 246176512, + "pass_at_k": { + "pass@1": 0.8260879333759387, + "pass@2": 0.8676958919313628, + "pass@4": 0.8980069189605794, + "pass@8": 0.9198953723230876, + "pass@16": 0.93505318616042, + "pass@32": 0.94562924717024, + "pass@64": 0.9530547716086694, + "pass@128": 0.9583000479309793 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465951492537313, + "pass@2": 0.9632176225173348, + "pass@4": 0.9735499216515845, + "pass@8": 0.9811835925161799, + "pass@16": 0.9873330366748251, + "pass@32": 0.9918809176478053, + "pass@64": 0.993806150809024, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981953581871345, + "pass@2": 0.9990279792328591, + "pass@4": 0.9996846265927285, + "pass@8": 0.9999691763603531, + "pass@16": 0.9999997893064115, + "pass@32": 0.9999999999982297, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9918198529411765, + "pass@2": 0.9951822313571099, + "pass@4": 0.9976438364322108, + "pass@8": 0.9994046739409391, + "pass@16": 0.9999673976331993, + "pass@32": 0.9999999551856581, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.993764787066246, + "pass@2": 0.9947496212027123, + "pass@4": 0.9955808440940151, + "pass@8": 0.9967326641590364, + "pass@16": 0.9979618214750134, + "pass@32": 0.9990096140434843, + "pass@64": 0.9998121152924854, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9986505469450812, + "pass@4": 0.9995743400943553, + "pass@8": 0.9999481010918784, + "pass@16": 0.9999994169232284, + "pass@32": 0.9999999999832521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9844923048048048, + "pass@2": 0.9919176065829609, + "pass@4": 0.9963254145596664, + "pass@8": 0.9990655284449819, + "pass@16": 0.9999326120862155, + "pass@32": 0.9999996834444386, + "pass@64": 0.999999999999752, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9835962620578779, + "pass@2": 0.9923585968554577, + "pass@4": 0.9956467118660005, + "pass@8": 0.9975453735871543, + "pass@16": 0.9990224437782512, + "pass@32": 0.9998343032263491, + "pass@64": 0.9999978524135079, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9548611111111112, + "pass@2": 0.9714758311461071, + "pass@4": 0.9792459415787311, + "pass@8": 0.9832824257084963, + "pass@16": 0.9859409115458316, + "pass@32": 0.9888924101781106, + "pass@64": 0.9916644760868305, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8987241972477065, + "pass@2": 0.9262153404849626, + "pass@4": 0.9399879209479548, + "pass@8": 0.9515969150881995, + "pass@16": 0.9632921895262051, + "pass@32": 0.9727991681405372, + "pass@64": 0.978484053984944, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7942467206790124, + "pass@2": 0.8419292098036356, + "pass@4": 0.87347388607674, + "pass@8": 0.896955802177486, + "pass@16": 0.9148879344767462, + "pass@32": 0.9305686019139384, + "pass@64": 0.9444548045895704, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7969445474777448, + "pass@2": 0.8519073167597371, + "pass@4": 0.8856038931706228, + "pass@8": 0.9087465231648425, + "pass@16": 0.9243278096477272, + "pass@32": 0.9348198946343538, + "pass@64": 0.9430984318055536, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.7702064043209876, + "pass@2": 0.8314692542772435, + "pass@4": 0.8710929519226761, + "pass@8": 0.9004926868932659, + "pass@16": 0.9209909756435962, + "pass@32": 0.9332392643870973, + "pass@64": 0.9400295244557415, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.6661366150442478, + "pass@2": 0.7643184708382698, + "pass@4": 0.8284491338472064, + "pass@8": 0.8694132293085024, + "pass@16": 0.8955625569041571, + "pass@32": 0.9130243917210557, + "pass@64": 0.926146214710774, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.5658179593373494, + "pass@2": 0.6702137042263541, + "pass@4": 0.7485751202221707, + "pass@8": 0.8014946697752476, + "pass@16": 0.8341101603053381, + "pass@32": 0.8534621646954216, + "pass@64": 0.8669435177559548, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.55859375, + "pass@2": 0.6654674403098803, + "pass@4": 0.7465657682910599, + "pass@8": 0.801548579552477, + "pass@16": 0.8354467036205091, + "pass@32": 0.8565727072192222, + "pass@64": 0.8703706077422788, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.3890436178247734, + "pass@2": 0.5075246211670672, + "pass@4": 0.6101444158604039, + "pass@8": 0.6887828332064626, + "pass@16": 0.7462233374111003, + "pass@32": 0.7901599832812849, + "pass@64": 0.8223790409544078, + "pass@128": 0.8429003021148036 + }, + "20": { + "pass@1": 0.39846096096096095, + "pass@2": 0.5144094635974552, + "pass@4": 0.6162668182805975, + "pass@8": 0.6980383217113035, + "pass@16": 0.7577338561538128, + "pass@32": 0.799757548916951, + "pass@64": 0.828102483308844, + "pass@128": 0.8468468468468469 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..78305db16e7e24d815014e54f7f596ae4eaa153a --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b21f23cf7c95aaee22d5a011127f91f6e887a3096a79c85a40541c898f663946 +size 2710541884 diff --git a/results/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv b/results/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..59937997aaa30eb24560a4e8709d2d1037291429 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op17-20,0.9890277777777777,0.6918046875,0.832594572368421,0.0,0.0,0.0,199.14139930555555,372.07711328125,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch4-op17-20/summary.json b/results/composition/SDAR-100M-block16-epoch4-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fcf3d5a19166901840773a61aef162e566f46786 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op17-20", + "id_acc": 0.9890277777777777, + "ood_acc": 0.6918046875, + "total_acc": 0.832594572368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.14139930555555, + "ood_avg_resp_len": 372.07711328125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 676633, + "answer_accuracy": 0.8356299893297502, + "avg_response_len": 276.8894900509801, + "resp_tokens_sum": 224205173, + "pass_at_k": { + "pass@1": 0.8356299893297502, + "pass@2": 0.8748209761426936, + "pass@4": 0.9020187386472343, + "pass@8": 0.9209896497859279, + "pass@16": 0.9346778150804824, + "pass@32": 0.9451618942195561, + "pass@64": 0.9534618266585658, + "pass@128": 0.9603224786595005 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9463956489675516, + "pass@2": 0.9599948029173347, + "pass@4": 0.9696439955514411, + "pass@8": 0.9772987780870966, + "pass@16": 0.9829576313879089, + "pass@32": 0.9869532680938505, + "pass@64": 0.9904289041602642, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962890625, + "pass@2": 0.9968965305118112, + "pass@4": 0.9969725551884139, + "pass@8": 0.9970703124996394, + "pass@16": 0.997265625, + "pass@32": 0.99765625, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941220238095239, + "pass@2": 0.9976346706661666, + "pass@4": 0.9991279667422525, + "pass@8": 0.999810090374339, + "pass@16": 0.9999911414698637, + "pass@32": 0.9999999920410901, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9975673652694611, + "pass@2": 0.9990827891461174, + "pass@4": 0.9997700759897528, + "pass@8": 0.9999845197580833, + "pass@16": 0.9999999528630369, + "pass@32": 0.9999999999999486, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960012943786982, + "pass@2": 0.9984763022410661, + "pass@4": 0.9996465231971745, + "pass@8": 0.9999815406455984, + "pass@16": 0.9999999678464966, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963441506410257, + "pass@2": 0.9996159209065214, + "pass@4": 0.9999852656639074, + "pass@8": 0.9999999592877328, + "pass@16": 0.9999999999999242, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9913292253521127, + "pass@2": 0.9970926444493733, + "pass@4": 0.9988132281880259, + "pass@8": 0.9997344814601936, + "pass@16": 0.9999856314389629, + "pass@32": 0.9999999749550461, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9559055815172538, + "pass@4": 0.9662483132420059, + "pass@8": 0.9743145176794873, + "pass@16": 0.9814528957415205, + "pass@32": 0.9870929428344121, + "pass@64": 0.9904799574420998, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8917284604519774, + "pass@2": 0.9156980542951199, + "pass@4": 0.9281065293215466, + "pass@8": 0.935889004485147, + "pass@16": 0.9418117371185404, + "pass@32": 0.947600022930526, + "pass@64": 0.9542037056880797, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8402233542319749, + "pass@2": 0.8852274269987415, + "pass@4": 0.9120427055748123, + "pass@8": 0.9300158215455117, + "pass@16": 0.9440746211517979, + "pass@32": 0.9571960077579881, + "pass@64": 0.9683369401551648, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8298773364485982, + "pass@2": 0.8854661092305048, + "pass@4": 0.9165421769708681, + "pass@8": 0.9342494936091128, + "pass@16": 0.9469401578089754, + "pass@32": 0.9577873713575069, + "pass@64": 0.9667747235355996, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7909365968208093, + "pass@2": 0.8548815057348322, + "pass@4": 0.8965234123552469, + "pass@8": 0.9232631518428708, + "pass@16": 0.93956332213618, + "pass@32": 0.9494509870773286, + "pass@64": 0.9560635736921728, + "pass@128": 0.9624277456647399 + }, + "16": { + "pass@1": 0.6512096774193549, + "pass@2": 0.7479818865887731, + "pass@4": 0.8152397825271837, + "pass@8": 0.8589674413513186, + "pass@16": 0.8885187025219686, + "pass@32": 0.9122664817583825, + "pass@64": 0.9310470742858282, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6124909682080925, + "pass@2": 0.7055230474261528, + "pass@4": 0.768419161297034, + "pass@8": 0.8132989938840093, + "pass@16": 0.8486630253769624, + "pass@32": 0.8756942867659188, + "pass@64": 0.8953656362346946, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5464732142857143, + "pass@2": 0.6480627812148483, + "pass@4": 0.7236771966896989, + "pass@8": 0.777945603229985, + "pass@16": 0.8163166798894307, + "pass@32": 0.8431802277020857, + "pass@64": 0.8622731277447071, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.4315643768768769, + "pass@2": 0.5412484334728428, + "pass@4": 0.6303853890448375, + "pass@8": 0.6978895463021549, + "pass@16": 0.7471806522306299, + "pass@32": 0.7842191914351475, + "pass@64": 0.8144645683656517, + "pass@128": 0.8438438438438438 + }, + "20": { + "pass@1": 0.4473066816816817, + "pass@2": 0.550585157007401, + "pass@4": 0.6322523291007539, + "pass@8": 0.6917831399797562, + "pass@16": 0.7350186424328486, + "pass@32": 0.7688185050754536, + "pass@64": 0.7971501932040521, + "pass@128": 0.8228228228228228 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 686415, + "answer_accuracy": 0.8359496784879189, + "avg_response_len": 286.5450993764614, + "resp_tokens_sum": 235287912, + "pass_at_k": { + "pass@1": 0.8359496784879189, + "pass@2": 0.8760342317157656, + "pass@4": 0.9043035755711732, + "pass@8": 0.9240919493445696, + "pass@16": 0.9381619892173285, + "pass@32": 0.9489126547729012, + "pass@64": 0.9576937527310582, + "pass@128": 0.9649259547934529 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924079754601227, + "pass@2": 0.9425480653108548, + "pass@4": 0.9558939233439372, + "pass@8": 0.9660136894297928, + "pass@16": 0.9745849213182081, + "pass@32": 0.9815053815983932, + "pass@64": 0.9856196459168264, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920969202898551, + "pass@2": 0.9947952327969871, + "pass@4": 0.9963463162213417, + "pass@8": 0.9977460600531718, + "pass@16": 0.9990916148451592, + "pass@32": 0.9998484564379228, + "pass@64": 0.999998063383779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966728617196489, + "pass@4": 0.9972907742728435, + "pass@8": 0.997639465969732, + "pass@16": 0.9980869583411133, + "pass@32": 0.9988007408816018, + "pass@64": 0.9996502944295288, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9983024498202175, + "pass@4": 0.9990612040546953, + "pass@8": 0.9997078494159238, + "pass@16": 0.999975444312545, + "pass@32": 0.9999999133144741, + "pass@64": 0.9999999999999895, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918573943661971, + "pass@2": 0.9956231978485083, + "pass@4": 0.9969242494336096, + "pass@8": 0.9971736180162805, + "pass@16": 0.9971830890857465, + "pass@32": 0.9971830985915483, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9910179640718563, + "pass@2": 0.9937379178650573, + "pass@4": 0.9939987547402381, + "pass@8": 0.9940119380249567, + "pass@16": 0.9940119760478334, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9533838478915663, + "pass@2": 0.9686280802343229, + "pass@4": 0.9774122846842942, + "pass@8": 0.9826945678160793, + "pass@16": 0.9851505016991625, + "pass@32": 0.9862401453810479, + "pass@64": 0.9872047186881181, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.885384012539185, + "pass@2": 0.9101669525831221, + "pass@4": 0.9239799511738148, + "pass@8": 0.9333532233276649, + "pass@16": 0.9404962983721141, + "pass@32": 0.947095869663446, + "pass@64": 0.9545776664683873, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8328956582633054, + "pass@2": 0.8768782119146881, + "pass@4": 0.9001525788793208, + "pass@8": 0.9144132901889223, + "pass@16": 0.9252804791130302, + "pass@32": 0.9358350221164605, + "pass@64": 0.9474943865376289, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.8158351608187134, + "pass@2": 0.8764760182115391, + "pass@4": 0.9120771337245117, + "pass@8": 0.9326000038402523, + "pass@16": 0.9449230559731131, + "pass@32": 0.9539728291687525, + "pass@64": 0.9628130945239121, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.7901515151515152, + "pass@2": 0.8613483506323071, + "pass@4": 0.9039200494824501, + "pass@8": 0.9292158194336342, + "pass@16": 0.9452248133475442, + "pass@32": 0.9574867632372372, + "pass@64": 0.9678628579175225, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6649528133903134, + "pass@2": 0.7624254800681963, + "pass@4": 0.8277979675617472, + "pass@8": 0.8686706781122179, + "pass@16": 0.8961598925297205, + "pass@32": 0.9179229188572494, + "pass@64": 0.9337860632473033, + "pass@128": 0.9430199430199431 + }, + "17": { + "pass@1": 0.6393390916149069, + "pass@2": 0.7354016634469608, + "pass@4": 0.8004865514675568, + "pass@8": 0.8453401336925654, + "pass@16": 0.8772971858982275, + "pass@32": 0.9004220002674332, + "pass@64": 0.9174945886997983, + "pass@128": 0.9316770186335404 + }, + "18": { + "pass@1": 0.5892693014705882, + "pass@2": 0.6895900880037054, + "pass@4": 0.7655650637420315, + "pass@8": 0.8203016454124229, + "pass@16": 0.859618198955504, + "pass@32": 0.8878937646484227, + "pass@64": 0.9097340634600377, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.4204799107142857, + "pass@2": 0.5389387361736032, + "pass@4": 0.6410350558858714, + "pass@8": 0.7190375968217035, + "pass@16": 0.7736645475875377, + "pass@32": 0.8123192037757934, + "pass@64": 0.8394299819521862, + "pass@128": 0.8571428571428571 + }, + "20": { + "pass@1": 0.3888005239520958, + "pass@2": 0.4932697351360271, + "pass@4": 0.581708334175294, + "pass@8": 0.6533021743606491, + "pass@16": 0.7090402348266315, + "pass@32": 0.754332532451074, + "pass@64": 0.7954232145576876, + "pass@128": 0.8293413173652695 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 661822, + "answer_accuracy": 0.8260879333759387, + "avg_response_len": 307.27815945039146, + "resp_tokens_sum": 246176512, + "pass_at_k": { + "pass@1": 0.8260879333759387, + "pass@2": 0.8676958919313628, + "pass@4": 0.8980069189605794, + "pass@8": 0.9198953723230876, + "pass@16": 0.93505318616042, + "pass@32": 0.94562924717024, + "pass@64": 0.9530547716086694, + "pass@128": 0.9583000479309793 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465951492537313, + "pass@2": 0.9632176225173348, + "pass@4": 0.9735499216515845, + "pass@8": 0.9811835925161799, + "pass@16": 0.9873330366748251, + "pass@32": 0.9918809176478053, + "pass@64": 0.993806150809024, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981953581871345, + "pass@2": 0.9990279792328591, + "pass@4": 0.9996846265927285, + "pass@8": 0.9999691763603531, + "pass@16": 0.9999997893064115, + "pass@32": 0.9999999999982297, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9918198529411765, + "pass@2": 0.9951822313571099, + "pass@4": 0.9976438364322108, + "pass@8": 0.9994046739409391, + "pass@16": 0.9999673976331993, + "pass@32": 0.9999999551856581, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.993764787066246, + "pass@2": 0.9947496212027123, + "pass@4": 0.9955808440940151, + "pass@8": 0.9967326641590364, + "pass@16": 0.9979618214750134, + "pass@32": 0.9990096140434843, + "pass@64": 0.9998121152924854, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9986505469450812, + "pass@4": 0.9995743400943553, + "pass@8": 0.9999481010918784, + "pass@16": 0.9999994169232284, + "pass@32": 0.9999999999832521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9844923048048048, + "pass@2": 0.9919176065829609, + "pass@4": 0.9963254145596664, + "pass@8": 0.9990655284449819, + "pass@16": 0.9999326120862155, + "pass@32": 0.9999996834444386, + "pass@64": 0.999999999999752, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9835962620578779, + "pass@2": 0.9923585968554577, + "pass@4": 0.9956467118660005, + "pass@8": 0.9975453735871543, + "pass@16": 0.9990224437782512, + "pass@32": 0.9998343032263491, + "pass@64": 0.9999978524135079, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9548611111111112, + "pass@2": 0.9714758311461071, + "pass@4": 0.9792459415787311, + "pass@8": 0.9832824257084963, + "pass@16": 0.9859409115458316, + "pass@32": 0.9888924101781106, + "pass@64": 0.9916644760868305, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8987241972477065, + "pass@2": 0.9262153404849626, + "pass@4": 0.9399879209479548, + "pass@8": 0.9515969150881995, + "pass@16": 0.9632921895262051, + "pass@32": 0.9727991681405372, + "pass@64": 0.978484053984944, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7942467206790124, + "pass@2": 0.8419292098036356, + "pass@4": 0.87347388607674, + "pass@8": 0.896955802177486, + "pass@16": 0.9148879344767462, + "pass@32": 0.9305686019139384, + "pass@64": 0.9444548045895704, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7969445474777448, + "pass@2": 0.8519073167597371, + "pass@4": 0.8856038931706228, + "pass@8": 0.9087465231648425, + "pass@16": 0.9243278096477272, + "pass@32": 0.9348198946343538, + "pass@64": 0.9430984318055536, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.7702064043209876, + "pass@2": 0.8314692542772435, + "pass@4": 0.8710929519226761, + "pass@8": 0.9004926868932659, + "pass@16": 0.9209909756435962, + "pass@32": 0.9332392643870973, + "pass@64": 0.9400295244557415, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.6661366150442478, + "pass@2": 0.7643184708382698, + "pass@4": 0.8284491338472064, + "pass@8": 0.8694132293085024, + "pass@16": 0.8955625569041571, + "pass@32": 0.9130243917210557, + "pass@64": 0.926146214710774, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.5658179593373494, + "pass@2": 0.6702137042263541, + "pass@4": 0.7485751202221707, + "pass@8": 0.8014946697752476, + "pass@16": 0.8341101603053381, + "pass@32": 0.8534621646954216, + "pass@64": 0.8669435177559548, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.55859375, + "pass@2": 0.6654674403098803, + "pass@4": 0.7465657682910599, + "pass@8": 0.801548579552477, + "pass@16": 0.8354467036205091, + "pass@32": 0.8565727072192222, + "pass@64": 0.8703706077422788, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.3890436178247734, + "pass@2": 0.5075246211670672, + "pass@4": 0.6101444158604039, + "pass@8": 0.6887828332064626, + "pass@16": 0.7462233374111003, + "pass@32": 0.7901599832812849, + "pass@64": 0.8223790409544078, + "pass@128": 0.8429003021148036 + }, + "20": { + "pass@1": 0.39846096096096095, + "pass@2": 0.5144094635974552, + "pass@4": 0.6162668182805975, + "pass@8": 0.6980383217113035, + "pass@16": 0.7577338561538128, + "pass@32": 0.799757548916951, + "pass@64": 0.828102483308844, + "pass@128": 0.8468468468468469 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..dc3a196da4e435b87c1f1be1bd569b98ff98bd46 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb818e9d3c62454f3489e1980f7b823cc05ceb82fbc59af63f720aa2d349199c +size 1789415021 diff --git a/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json b/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2a39e8f2e3e260ac02ea29c2cfc53d6999363371 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op7-10", + "id": { + "answer_accuracy": 0.9928402777777777, + "avg_response_len": 200.1375373263889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9928402777777777, + "pass@2": 0.9949716207349067, + "pass@4": 0.9962314358309377, + "pass@8": 0.9970589677382524, + "pass@16": 0.997637285152463, + "pass@32": 0.9980942875801967, + "pass@64": 0.9984440103790677, + "pass@128": 0.9986666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954203125, + "pass@2": 0.9668100393700787, + "pass@4": 0.9751169003562058, + "pass@8": 0.9805623138162619, + "pass@16": 0.9841740075635769, + "pass@32": 0.9868633714733398, + "pass@64": 0.9886806402008643, + "pass@128": 0.99 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990234375, + "pass@2": 0.999410063976378, + "pass@4": 0.9996751137982751, + "pass@8": 0.9998989158969318, + "pass@16": 0.9999915037321406, + "pass@32": 0.9999999700068081, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949921875, + "pass@2": 0.9967944143700785, + "pass@4": 0.9977303270528682, + "pass@8": 0.9982286507773027, + "pass@16": 0.9986409291833077, + "pass@32": 0.999127515746347, + "pass@64": 0.9996924094487892, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986875, + "pass@2": 0.9990652066929134, + "pass@4": 0.9992026385451819, + "pass@8": 0.99937067318584, + "pass@16": 0.9996167406956445, + "pass@32": 0.9998739053984705, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977421875, + "pass@2": 0.9986300442913385, + "pass@4": 0.9992237436257967, + "pass@8": 0.999713898666317, + "pass@16": 0.9999638315713223, + "pass@32": 0.9999996347541911, + "pass@64": 0.9999999999983727, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968125, + "pass@2": 0.998474655511811, + "pass@4": 0.9989533392388451, + "pass@8": 0.9989996318701388, + "pass@16": 0.9989999999891657, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.994109375, + "pass@2": 0.9955601624015749, + "pass@4": 0.9961808598612675, + "pass@8": 0.9967566254314778, + "pass@16": 0.9973485536370097, + "pass@32": 0.9979841908426146, + "pass@64": 0.9986296156860006, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.954203125, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9990234375, + "5": 0.9949921875, + "6": 0.9986875, + "7": 0.9977421875, + "8": 0.9968125, + "9": 0.994109375 + }, + "per_op_avg_response_len": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376709, + "answer_accuracy": 0.993934164978048, + "avg_response_len": 190.66135279466397, + "resp_tokens_sum": 72262178, + "pass_at_k": { + "pass@1": 0.993934164978048, + "pass@2": 0.9959355978109121, + "pass@4": 0.9969234930043573, + "pass@8": 0.9975032519300019, + "pass@16": 0.9979521898023057, + "pass@32": 0.998386677156877, + "pass@64": 0.9988549563110108, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639564896755162, + "pass@2": 0.9751933663159359, + "pass@4": 0.9805886808396739, + "pass@8": 0.9832729426842688, + "pass@16": 0.9853094435850058, + "pass@32": 0.9875159779599025, + "pass@64": 0.9903780082743677, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.997314453125, + "pass@2": 0.9981799028051181, + "pass@4": 0.9989848275801462, + "pass@8": 0.9996841121782726, + "pass@16": 0.9999734491629393, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9964285714285714, + "pass@2": 0.9987411729783777, + "pass@4": 0.9997791192767571, + "pass@8": 0.9999910052051663, + "pass@16": 0.999999987524699, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9973235124239709, + "pass@4": 0.9976157328200742, + "pass@8": 0.9981157895218378, + "pass@16": 0.998852517052828, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972725591715976, + "pass@2": 0.9980547919675722, + "pass@4": 0.9987735604794961, + "pass@8": 0.9995079830474547, + "pass@16": 0.999928825622786, + "pass@32": 0.9999991140832827, + "pass@64": 0.9999999999952738, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9991486378205128, + "pass@2": 0.9999641158136482, + "pass@4": 0.999999888836011, + "pass@8": 0.9999999999996302, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955325704225352, + "pass@2": 0.9969474326272597, + "pass@4": 0.9973978846834287, + "pass@8": 0.9976821039095084, + "pass@16": 0.998119293157022, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387445, + "answer_accuracy": 0.9917804923001311, + "avg_response_len": 198.17119921363042, + "resp_tokens_sum": 77416768, + "pass_at_k": { + "pass@1": 0.9917804923001311, + "pass@2": 0.9938671701659427, + "pass@4": 0.9952572525235622, + "pass@8": 0.996152824229983, + "pass@16": 0.9966572318246322, + "pass@32": 0.9970099127946118, + "pass@64": 0.9972736781091658, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412384969325154, + "pass@2": 0.9570337030819772, + "pass@4": 0.9684386097635029, + "pass@8": 0.9755421990555402, + "pass@16": 0.9790090135040668, + "pass@32": 0.9808182150392867, + "pass@64": 0.9815547734367644, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963768115942029, + "pass@2": 0.9971339010612804, + "pass@4": 0.9974446123582379, + "pass@8": 0.9977680501821345, + "pass@16": 0.9983124881128967, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999841363400491, + "pass@4": 0.9999999435958757, + "pass@8": 0.9999999999999097, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997674313583815, + "pass@2": 0.998275069978608, + "pass@4": 0.9989801556597332, + "pass@8": 0.9996539139236084, + "pass@16": 0.9999649956979594, + "pass@32": 0.9999998098093686, + "pass@64": 0.9999999999999141, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957306338028169, + "pass@2": 0.9968081124542529, + "pass@4": 0.9971466248937193, + "pass@8": 0.9971827493473799, + "pass@16": 0.9971830985789736, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.993310254491018, + "pass@2": 0.9940558106228489, + "pass@4": 0.994196379457059, + "pass@8": 0.9943759135003243, + "pass@16": 0.9947162761091989, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379598, + "answer_accuracy": 0.9928387596250419, + "avg_response_len": 211.5403650192501, + "resp_tokens_sum": 80879497, + "pass_at_k": { + "pass@1": 0.9928387596250419, + "pass@2": 0.9951445189653858, + "pass@4": 0.9965407850654927, + "pass@8": 0.9975444128321481, + "pass@16": 0.9983265017873332, + "pass@32": 0.9989124149016076, + "pass@64": 0.9992324413075104, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569496268656716, + "pass@2": 0.9678403014455282, + "pass@4": 0.976078659010907, + "pass@8": 0.9827045652363846, + "pass@16": 0.9880512471221918, + "pass@32": 0.9920857219167919, + "pass@64": 0.9938974306133376, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997487207602339, + "pass@2": 0.9999802142330892, + "pass@4": 0.9999999095507799, + "pass@8": 0.9999999999996626, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9922564338235295, + "pass@2": 0.994646320634553, + "pass@4": 0.9961220976973466, + "pass@8": 0.9970632553671733, + "pass@16": 0.997715072688465, + "pass@32": 0.9983497369764539, + "pass@64": 0.9992704955997272, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992360015772871, + "pass@2": 0.9998886118631859, + "pass@4": 0.9999968563669289, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998318829113924, + "pass@2": 0.9996340202332306, + "pass@4": 0.9999719820425929, + "pass@8": 0.9999998296794577, + "pass@16": 0.999999999997869, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995777027027027, + "pass@2": 0.9988557701402191, + "pass@4": 0.9998988651193375, + "pass@8": 0.9999992668226257, + "pass@16": 0.9999999999808709, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9933430466237942, + "pass@2": 0.9955922285236853, + "pass@4": 0.996922894083577, + "pass@8": 0.99825698853535, + "pass@16": 0.9992957181536803, + "pass@32": 0.9998834594758289, + "pass@64": 0.9999990174474539, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4674421875, + "avg_response_len": 332.59690703125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4674421875, + "pass@2": 0.5340498277559019, + "pass@4": 0.5905295806805306, + "pass@8": 0.6377652123780675, + "pass@16": 0.6765026264659709, + "pass@32": 0.7087523766471188, + "pass@64": 0.7369760709987055, + "pass@128": 0.7626 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9424921875, + "pass@2": 0.9608925935039377, + "pass@4": 0.9707150841769778, + "pass@8": 0.9776590164210477, + "pass@16": 0.983031112952659, + "pass@32": 0.9868108166417683, + "pass@64": 0.9895059512740915, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.850125, + "pass@2": 0.8868829970472448, + "pass@4": 0.9069662009748772, + "pass@8": 0.9210538138715969, + "pass@16": 0.932209399638482, + "pass@32": 0.941670117173944, + "pass@64": 0.9509294871368753, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.723671875, + "pass@2": 0.7991273375984257, + "pass@4": 0.848705464191976, + "pass@8": 0.8832906602998529, + "pass@16": 0.907676298226515, + "pass@32": 0.9258390025587503, + "pass@64": 0.9408623838730613, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.621359375, + "pass@2": 0.7092471702755899, + "pass@4": 0.7710694283839518, + "pass@8": 0.8143357774146883, + "pass@16": 0.8459435216064888, + "pass@32": 0.8706776031668482, + "pass@64": 0.8916396601771912, + "pass@128": 0.911 + }, + "15": { + "pass@1": 0.4664140625, + "pass@2": 0.5631503444881897, + "pass@4": 0.6409323717660281, + "pass@8": 0.700385082200824, + "pass@16": 0.7464661637312656, + "pass@32": 0.7838249784646978, + "pass@64": 0.8150213816108757, + "pass@128": 0.842 + }, + "16": { + "pass@1": 0.300421875, + "pass@2": 0.38325036909448806, + "pass@4": 0.45934014332583345, + "pass@8": 0.5256147490839751, + "pass@16": 0.5812667481374817, + "pass@32": 0.6290118259853492, + "pass@64": 0.6714562961595262, + "pass@128": 0.71 + }, + "17": { + "pass@1": 0.2392421875, + "pass@2": 0.3153343996062992, + "pass@4": 0.390559457255343, + "pass@8": 0.45835220794408055, + "pass@16": 0.515216747490821, + "pass@32": 0.5633337200265438, + "pass@64": 0.6057651819056846, + "pass@128": 0.644 + }, + "18": { + "pass@1": 0.20425, + "pass@2": 0.27352423720472446, + "pass@4": 0.3419236596362953, + "pass@8": 0.4029165957544186, + "pass@16": 0.45499798232417954, + "pass@32": 0.5004840348220753, + "pass@64": 0.5415533976453777, + "pass@128": 0.578 + }, + "19": { + "pass@1": 0.169625, + "pass@2": 0.23175504429133828, + "pass@4": 0.29506946662917083, + "pass@8": 0.353428445257125, + "pass@16": 0.40308684493744273, + "pass@32": 0.4453773587288245, + "pass@64": 0.4839624351859198, + "pass@128": 0.52 + }, + "20": { + "pass@1": 0.1568203125, + "pass@2": 0.217333784448819, + "pass@4": 0.2800145304649416, + "pass@8": 0.3406157755330686, + "pass@16": 0.39513144561425667, + "pass@32": 0.4404943089023678, + "pass@64": 0.47906453501838553, + "pass@128": 0.514 + } + }, + "per_op_accuracy": { + "11": 0.9424921875, + "12": 0.850125, + "13": 0.723671875, + "14": 0.621359375, + "15": 0.4664140625, + "16": 0.300421875, + "17": 0.2392421875, + "18": 0.20425, + "19": 0.169625, + "20": 0.1568203125 + }, + "per_op_avg_response_len": { + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 200217, + "answer_accuracy": 0.46484258915304605, + "avg_response_len": 315.6339617384844, + "resp_tokens_sum": 135949860, + "pass_at_k": { + "pass@1": 0.46484258915304605, + "pass@2": 0.5302466260486013, + "pass@4": 0.5859201473515673, + "pass@8": 0.6324395587767444, + "pass@16": 0.6708254517301923, + "pass@32": 0.7030991125764335, + "pass@64": 0.7314564132966396, + "pass@128": 0.7575037147102526 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9345122167138811, + "pass@2": 0.9542368143695208, + "pass@4": 0.9657462063877991, + "pass@8": 0.9749512748470323, + "pass@16": 0.9827357531203197, + "pass@32": 0.9884344587547554, + "pass@64": 0.9925691216469926, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.8454493290960452, + "pass@2": 0.8801169285777846, + "pass@4": 0.8989873583069065, + "pass@8": 0.9130351649779792, + "pass@16": 0.9250958046726186, + "pass@32": 0.9364960935539219, + "pass@64": 0.9474650567648585, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.74047315830721, + "pass@2": 0.8119554217164869, + "pass@4": 0.8599147251969675, + "pass@8": 0.8945134207641785, + "pass@16": 0.9205661933194598, + "pass@32": 0.9404403427759488, + "pass@64": 0.9559823521026871, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6125632788161994, + "pass@2": 0.7013274370201388, + "pass@4": 0.7658476230190846, + "pass@8": 0.8097702298192307, + "pass@16": 0.8407102562368179, + "pass@32": 0.8644964958281782, + "pass@64": 0.8838021812976606, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.48457821531791906, + "pass@2": 0.5827305027081152, + "pass@4": 0.6590647869702699, + "pass@8": 0.7158310076759379, + "pass@16": 0.7589664128271321, + "pass@32": 0.7930825505983045, + "pass@64": 0.8204516163076292, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.2755040322580645, + "pass@2": 0.3586213328676657, + "pass@4": 0.43517604109970115, + "pass@8": 0.5017510566263869, + "pass@16": 0.5583572473734497, + "pass@32": 0.6078831915021241, + "pass@64": 0.6517453480905581, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.21780166184971098, + "pass@2": 0.29274348857584986, + "pass@4": 0.36758594012453627, + "pass@8": 0.4327260544001389, + "pass@16": 0.4852625887317101, + "pass@32": 0.5308733536032256, + "pass@64": 0.5731955827266514, + "pass@128": 0.615606936416185 + }, + "18": { + "pass@1": 0.18082589285714284, + "pass@2": 0.24475007030371193, + "pass@4": 0.30927755396646855, + "pass@8": 0.3685482161042458, + "pass@16": 0.4204588630253465, + "pass@32": 0.46585847061159913, + "pass@64": 0.5068792519157992, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.1753472222222222, + "pass@2": 0.23986560379276908, + "pass@4": 0.3042127763308865, + "pass@8": 0.3648899684541569, + "pass@16": 0.41793983675736257, + "pass@32": 0.46237397780199607, + "pass@64": 0.5029501083188042, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.2165424528859568, + "pass@4": 0.27821547475484476, + "pass@8": 0.3365076147164066, + "pass@16": 0.3888313889531125, + "pass@32": 0.4337362063462613, + "pass@64": 0.4738117632886543, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 201848, + "answer_accuracy": 0.46890796907523047, + "avg_response_len": 328.5204500260184, + "resp_tokens_sum": 141416227, + "pass_at_k": { + "pass@1": 0.46890796907523047, + "pass@2": 0.5365449638961293, + "pass@4": 0.5941743382935776, + "pass@8": 0.6423905408438726, + "pass@16": 0.6812675313224216, + "pass@32": 0.7127574655170806, + "pass@64": 0.7399460641942278, + "pass@128": 0.7647933392804044 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9478068524096386, + "pass@2": 0.9611561403092685, + "pass@4": 0.9686419691702693, + "pass@8": 0.9750669382908692, + "pass@16": 0.9806081356024587, + "pass@32": 0.9842428892059527, + "pass@64": 0.9862549466314855, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8410315438871473, + "pass@2": 0.8755083244884359, + "pass@4": 0.8937380000305601, + "pass@8": 0.905965360565369, + "pass@16": 0.9151980908995769, + "pass@32": 0.9235809759059825, + "pass@64": 0.9339121914379174, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7262123599439776, + "pass@2": 0.8062757091025381, + "pass@4": 0.8566690912585506, + "pass@8": 0.8882637919932588, + "pass@16": 0.9082264797954909, + "pass@32": 0.9222958616323473, + "pass@64": 0.9343518871725501, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6329952485380117, + "pass@2": 0.7221707792282543, + "pass@4": 0.7828848704767167, + "pass@8": 0.8258118272832482, + "pass@16": 0.8579023706600551, + "pass@32": 0.8832258032244631, + "pass@64": 0.9049950902372479, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.4446022727272727, + "pass@2": 0.5437966028394173, + "pass@4": 0.6285262524002679, + "pass@8": 0.6960194300230498, + "pass@16": 0.7475932370874276, + "pass@32": 0.785985388651888, + "pass@64": 0.8147437752003887, + "pass@128": 0.8424242424242424 + }, + "16": { + "pass@1": 0.3059116809116809, + "pass@2": 0.39158751990936996, + "pass@4": 0.4727661555660242, + "pass@8": 0.5448737019577484, + "pass@16": 0.6040431432838912, + "pass@32": 0.6528764342680178, + "pass@64": 0.694521859735016, + "pass@128": 0.7293447293447294 + }, + "17": { + "pass@1": 0.26052989130434784, + "pass@2": 0.34320820291485293, + "pass@4": 0.42293633977041684, + "pass@8": 0.4927356348756277, + "pass@16": 0.5482837994495754, + "pass@32": 0.5921106992309274, + "pass@64": 0.6290502589670679, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.2153262867647059, + "pass@2": 0.28677049559981455, + "pass@4": 0.35674156300315396, + "pass@8": 0.4197868754911729, + "pass@16": 0.47482275196634255, + "pass@32": 0.5236327311772606, + "pass@64": 0.5672130434247589, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.16901506696428573, + "pass@2": 0.22576206294525675, + "pass@4": 0.28495160454273577, + "pass@8": 0.34010383127747634, + "pass@16": 0.38708927582856617, + "pass@32": 0.4276740630065036, + "pass@64": 0.4649332236307543, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.1504491017964072, + "pass@2": 0.2108255192371163, + "pass@4": 0.2735428047167756, + "pass@8": 0.3338204070406974, + "pass@16": 0.38676384570770866, + "pass@32": 0.4292259673414519, + "pass@64": 0.4663031080186876, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 196261, + "answer_accuracy": 0.4686091266809291, + "avg_response_len": 354.23182017878975, + "resp_tokens_sum": 148357954, + "pass_at_k": { + "pass@1": 0.4686091266809291, + "pass@2": 0.5353965975925558, + "pass@4": 0.5915239032047982, + "pass@8": 0.6384882700608202, + "pass@16": 0.6774437383099655, + "pass@32": 0.7104498460016812, + "pass@64": 0.7396000199751166, + "pass@128": 0.7655867970660146 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9458333333333333, + "pass@2": 0.9680735220597424, + "pass@4": 0.9784683795477946, + "pass@8": 0.9834253742459571, + "pass@16": 0.9859158447021265, + "pass@32": 0.9876978205871854, + "pass@64": 0.9894997112731746, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8640577217125383, + "pass@2": 0.9053041248284324, + "pass@4": 0.9285084223990344, + "pass@8": 0.9444538698747363, + "pass@16": 0.9565054855884125, + "pass@32": 0.9649179472227764, + "pass@64": 0.971280941814739, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7043306327160493, + "pass@2": 0.7786207713619127, + "pass@4": 0.8288944483791375, + "pass@8": 0.86676143621755, + "pass@16": 0.8943791026870926, + "pass@32": 0.9153670080570203, + "pass@64": 0.9331493513324162, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6179293397626113, + "pass@2": 0.703675538563985, + "pass@4": 0.7640525688183634, + "pass@8": 0.8070382454357383, + "pass@16": 0.8387920432900056, + "pass@32": 0.8638308703360127, + "pass@64": 0.8855514513931839, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.46923225308641975, + "pass@2": 0.5619528136239913, + "pass@4": 0.6342046055007012, + "pass@8": 0.6883368569053809, + "pass@16": 0.7319691872044063, + "pass@32": 0.7717383879708698, + "pass@64": 0.8095051745441575, + "pass@128": 0.8425925925925926 + }, + "16": { + "pass@1": 0.31752396755162243, + "pass@2": 0.3971402254198313, + "pass@4": 0.46753584065266157, + "pass@8": 0.5274963190047928, + "pass@16": 0.578633799879253, + "pass@32": 0.6236236229841198, + "pass@64": 0.6655990132285015, + "pass@128": 0.7079646017699115 + }, + "17": { + "pass@1": 0.2409403237951807, + "pass@2": 0.3118437084953988, + "pass@4": 0.38310006194632296, + "pass@8": 0.4517112008785566, + "pass@16": 0.5143629770086949, + "pass@32": 0.5692526943598468, + "pass@64": 0.6171244785989948, + "pass@128": 0.6566265060240963 + }, + "18": { + "pass@1": 0.21854838709677418, + "pass@2": 0.2914830454660909, + "pass@4": 0.36253027202244875, + "pass@8": 0.4232167175836581, + "pass@16": 0.4722504664411351, + "pass@32": 0.5141885209927322, + "pass@64": 0.5525587893884821, + "pass@128": 0.5870967741935483 + }, + "19": { + "pass@1": 0.1644873489425982, + "pass@2": 0.22967898815329338, + "pass@4": 0.296141610225459, + "pass@8": 0.35542356027993677, + "pass@16": 0.40438333117475245, + "pass@32": 0.44624875815883597, + "pass@64": 0.4841766947910102, + "pass@128": 0.5196374622356495 + }, + "20": { + "pass@1": 0.16366366366366367, + "pass@2": 0.22465292556335845, + "pass@4": 0.28830474653506144, + "pass@8": 0.3515397113540908, + "pass@16": 0.4098242301095972, + "pass@32": 0.4585545918829372, + "pass@64": 0.49711705635141545, + "pass@128": 0.5285285285285285 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7163149671052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7163149671052632, + "pass@2": 0.7523812033775175, + "pass@4": 0.7827041436464997, + "pass@8": 0.8079569912329014, + "pass@16": 0.8286190437385256, + "pass@32": 0.8458090712996409, + "pass@64": 0.8608293054420325, + "pass@128": 0.874421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954203125, + "pass@2": 0.9668100393700787, + "pass@4": 0.9751169003562058, + "pass@8": 0.9805623138162619, + "pass@16": 0.9841740075635769, + "pass@32": 0.9868633714733398, + "pass@64": 0.9886806402008643, + "pass@128": 0.99 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990234375, + "pass@2": 0.999410063976378, + "pass@4": 0.9996751137982751, + "pass@8": 0.9998989158969318, + "pass@16": 0.9999915037321406, + "pass@32": 0.9999999700068081, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949921875, + "pass@2": 0.9967944143700785, + "pass@4": 0.9977303270528682, + "pass@8": 0.9982286507773027, + "pass@16": 0.9986409291833077, + "pass@32": 0.999127515746347, + "pass@64": 0.9996924094487892, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986875, + "pass@2": 0.9990652066929134, + "pass@4": 0.9992026385451819, + "pass@8": 0.99937067318584, + "pass@16": 0.9996167406956445, + "pass@32": 0.9998739053984705, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977421875, + "pass@2": 0.9986300442913385, + "pass@4": 0.9992237436257967, + "pass@8": 0.999713898666317, + "pass@16": 0.9999638315713223, + "pass@32": 0.9999996347541911, + "pass@64": 0.9999999999983727, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968125, + "pass@2": 0.998474655511811, + "pass@4": 0.9989533392388451, + "pass@8": 0.9989996318701388, + "pass@16": 0.9989999999891657, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.994109375, + "pass@2": 0.9955601624015749, + "pass@4": 0.9961808598612675, + "pass@8": 0.9967566254314778, + "pass@16": 0.9973485536370097, + "pass@32": 0.9979841908426146, + "pass@64": 0.9986296156860006, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9424921875, + "pass@2": 0.9608925935039377, + "pass@4": 0.9707150841769778, + "pass@8": 0.9776590164210477, + "pass@16": 0.983031112952659, + "pass@32": 0.9868108166417683, + "pass@64": 0.9895059512740915, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.850125, + "pass@2": 0.8868829970472448, + "pass@4": 0.9069662009748772, + "pass@8": 0.9210538138715969, + "pass@16": 0.932209399638482, + "pass@32": 0.941670117173944, + "pass@64": 0.9509294871368753, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.723671875, + "pass@2": 0.7991273375984257, + "pass@4": 0.848705464191976, + "pass@8": 0.8832906602998529, + "pass@16": 0.907676298226515, + "pass@32": 0.9258390025587503, + "pass@64": 0.9408623838730613, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.621359375, + "pass@2": 0.7092471702755899, + "pass@4": 0.7710694283839518, + "pass@8": 0.8143357774146883, + "pass@16": 0.8459435216064888, + "pass@32": 0.8706776031668482, + "pass@64": 0.8916396601771912, + "pass@128": 0.911 + }, + "15": { + "pass@1": 0.4664140625, + "pass@2": 0.5631503444881897, + "pass@4": 0.6409323717660281, + "pass@8": 0.700385082200824, + "pass@16": 0.7464661637312656, + "pass@32": 0.7838249784646978, + "pass@64": 0.8150213816108757, + "pass@128": 0.842 + }, + "16": { + "pass@1": 0.300421875, + "pass@2": 0.38325036909448806, + "pass@4": 0.45934014332583345, + "pass@8": 0.5256147490839751, + "pass@16": 0.5812667481374817, + "pass@32": 0.6290118259853492, + "pass@64": 0.6714562961595262, + "pass@128": 0.71 + }, + "17": { + "pass@1": 0.2392421875, + "pass@2": 0.3153343996062992, + "pass@4": 0.390559457255343, + "pass@8": 0.45835220794408055, + "pass@16": 0.515216747490821, + "pass@32": 0.5633337200265438, + "pass@64": 0.6057651819056846, + "pass@128": 0.644 + }, + "18": { + "pass@1": 0.20425, + "pass@2": 0.27352423720472446, + "pass@4": 0.3419236596362953, + "pass@8": 0.4029165957544186, + "pass@16": 0.45499798232417954, + "pass@32": 0.5004840348220753, + "pass@64": 0.5415533976453777, + "pass@128": 0.578 + }, + "19": { + "pass@1": 0.169625, + "pass@2": 0.23175504429133828, + "pass@4": 0.29506946662917083, + "pass@8": 0.353428445257125, + "pass@16": 0.40308684493744273, + "pass@32": 0.4453773587288245, + "pass@64": 0.4839624351859198, + "pass@128": 0.52 + }, + "20": { + "pass@1": 0.1568203125, + "pass@2": 0.217333784448819, + "pass@4": 0.2800145304649416, + "pass@8": 0.3406157755330686, + "pass@16": 0.39513144561425667, + "pass@32": 0.4404943089023678, + "pass@64": 0.47906453501838553, + "pass@128": 0.514 + } + }, + "per_op_accuracy": { + "10": 0.954203125, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9990234375, + "5": 0.9949921875, + "6": 0.9986875, + "7": 0.9977421875, + "8": 0.9968125, + "9": 0.994109375, + "11": 0.9424921875, + "12": 0.850125, + "13": 0.723671875, + "14": 0.621359375, + "15": 0.4664140625, + "16": 0.300421875, + "17": 0.2392421875, + "18": 0.20425, + "19": 0.169625, + "20": 0.1568203125 + }, + "per_op_avg_response_len": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875, + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875, + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 576926, + "answer_accuracy": 0.7124935780904205, + "avg_response_len": 257.13824642349033, + "resp_tokens_sum": 208212038, + "pass_at_k": { + "pass@1": 0.7124935780904205, + "pass@2": 0.7482208665462601, + "pass@4": 0.7782977803705171, + "pass@8": 0.8033142972254913, + "pass@16": 0.8239431045015444, + "pass@32": 0.841313857869306, + "pass@64": 0.8566171919665101, + "pass@128": 0.8706923806512804 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639564896755162, + "pass@2": 0.9751933663159359, + "pass@4": 0.9805886808396739, + "pass@8": 0.9832729426842688, + "pass@16": 0.9853094435850058, + "pass@32": 0.9875159779599025, + "pass@64": 0.9903780082743677, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.997314453125, + "pass@2": 0.9981799028051181, + "pass@4": 0.9989848275801462, + "pass@8": 0.9996841121782726, + "pass@16": 0.9999734491629393, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9964285714285714, + "pass@2": 0.9987411729783777, + "pass@4": 0.9997791192767571, + "pass@8": 0.9999910052051663, + "pass@16": 0.999999987524699, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9973235124239709, + "pass@4": 0.9976157328200742, + "pass@8": 0.9981157895218378, + "pass@16": 0.998852517052828, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972725591715976, + "pass@2": 0.9980547919675722, + "pass@4": 0.9987735604794961, + "pass@8": 0.9995079830474547, + "pass@16": 0.999928825622786, + "pass@32": 0.9999991140832827, + "pass@64": 0.9999999999952738, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9991486378205128, + "pass@2": 0.9999641158136482, + "pass@4": 0.999999888836011, + "pass@8": 0.9999999999996302, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955325704225352, + "pass@2": 0.9969474326272597, + "pass@4": 0.9973978846834287, + "pass@8": 0.9976821039095084, + "pass@16": 0.998119293157022, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9345122167138811, + "pass@2": 0.9542368143695208, + "pass@4": 0.9657462063877991, + "pass@8": 0.9749512748470323, + "pass@16": 0.9827357531203197, + "pass@32": 0.9884344587547554, + "pass@64": 0.9925691216469926, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.8454493290960452, + "pass@2": 0.8801169285777846, + "pass@4": 0.8989873583069065, + "pass@8": 0.9130351649779792, + "pass@16": 0.9250958046726186, + "pass@32": 0.9364960935539219, + "pass@64": 0.9474650567648585, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.74047315830721, + "pass@2": 0.8119554217164869, + "pass@4": 0.8599147251969675, + "pass@8": 0.8945134207641785, + "pass@16": 0.9205661933194598, + "pass@32": 0.9404403427759488, + "pass@64": 0.9559823521026871, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6125632788161994, + "pass@2": 0.7013274370201388, + "pass@4": 0.7658476230190846, + "pass@8": 0.8097702298192307, + "pass@16": 0.8407102562368179, + "pass@32": 0.8644964958281782, + "pass@64": 0.8838021812976606, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.48457821531791906, + "pass@2": 0.5827305027081152, + "pass@4": 0.6590647869702699, + "pass@8": 0.7158310076759379, + "pass@16": 0.7589664128271321, + "pass@32": 0.7930825505983045, + "pass@64": 0.8204516163076292, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.2755040322580645, + "pass@2": 0.3586213328676657, + "pass@4": 0.43517604109970115, + "pass@8": 0.5017510566263869, + "pass@16": 0.5583572473734497, + "pass@32": 0.6078831915021241, + "pass@64": 0.6517453480905581, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.21780166184971098, + "pass@2": 0.29274348857584986, + "pass@4": 0.36758594012453627, + "pass@8": 0.4327260544001389, + "pass@16": 0.4852625887317101, + "pass@32": 0.5308733536032256, + "pass@64": 0.5731955827266514, + "pass@128": 0.615606936416185 + }, + "18": { + "pass@1": 0.18082589285714284, + "pass@2": 0.24475007030371193, + "pass@4": 0.30927755396646855, + "pass@8": 0.3685482161042458, + "pass@16": 0.4204588630253465, + "pass@32": 0.46585847061159913, + "pass@64": 0.5068792519157992, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.1753472222222222, + "pass@2": 0.23986560379276908, + "pass@4": 0.3042127763308865, + "pass@8": 0.3648899684541569, + "pass@16": 0.41793983675736257, + "pass@32": 0.46237397780199607, + "pass@64": 0.5029501083188042, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.2165424528859568, + "pass@4": 0.27821547475484476, + "pass@8": 0.3365076147164066, + "pass@16": 0.3888313889531125, + "pass@32": 0.4337362063462613, + "pass@64": 0.4738117632886543, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 589293, + "answer_accuracy": 0.71766976812159, + "avg_response_len": 266.5054985872954, + "resp_tokens_sum": 218832995, + "pass_at_k": { + "pass@1": 0.71766976812159, + "pass@2": 0.7541205482352454, + "pass@4": 0.7849935205585589, + "pass@8": 0.8106964627292033, + "pass@16": 0.831317315567593, + "pass@32": 0.8479933921096046, + "pass@64": 0.8623722337450352, + "pass@128": 0.875448168355417 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412384969325154, + "pass@2": 0.9570337030819772, + "pass@4": 0.9684386097635029, + "pass@8": 0.9755421990555402, + "pass@16": 0.9790090135040668, + "pass@32": 0.9808182150392867, + "pass@64": 0.9815547734367644, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963768115942029, + "pass@2": 0.9971339010612804, + "pass@4": 0.9974446123582379, + "pass@8": 0.9977680501821345, + "pass@16": 0.9983124881128967, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999841363400491, + "pass@4": 0.9999999435958757, + "pass@8": 0.9999999999999097, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997674313583815, + "pass@2": 0.998275069978608, + "pass@4": 0.9989801556597332, + "pass@8": 0.9996539139236084, + "pass@16": 0.9999649956979594, + "pass@32": 0.9999998098093686, + "pass@64": 0.9999999999999141, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957306338028169, + "pass@2": 0.9968081124542529, + "pass@4": 0.9971466248937193, + "pass@8": 0.9971827493473799, + "pass@16": 0.9971830985789736, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.993310254491018, + "pass@2": 0.9940558106228489, + "pass@4": 0.994196379457059, + "pass@8": 0.9943759135003243, + "pass@16": 0.9947162761091989, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9478068524096386, + "pass@2": 0.9611561403092685, + "pass@4": 0.9686419691702693, + "pass@8": 0.9750669382908692, + "pass@16": 0.9806081356024587, + "pass@32": 0.9842428892059527, + "pass@64": 0.9862549466314855, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8410315438871473, + "pass@2": 0.8755083244884359, + "pass@4": 0.8937380000305601, + "pass@8": 0.905965360565369, + "pass@16": 0.9151980908995769, + "pass@32": 0.9235809759059825, + "pass@64": 0.9339121914379174, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7262123599439776, + "pass@2": 0.8062757091025381, + "pass@4": 0.8566690912585506, + "pass@8": 0.8882637919932588, + "pass@16": 0.9082264797954909, + "pass@32": 0.9222958616323473, + "pass@64": 0.9343518871725501, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6329952485380117, + "pass@2": 0.7221707792282543, + "pass@4": 0.7828848704767167, + "pass@8": 0.8258118272832482, + "pass@16": 0.8579023706600551, + "pass@32": 0.8832258032244631, + "pass@64": 0.9049950902372479, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.4446022727272727, + "pass@2": 0.5437966028394173, + "pass@4": 0.6285262524002679, + "pass@8": 0.6960194300230498, + "pass@16": 0.7475932370874276, + "pass@32": 0.785985388651888, + "pass@64": 0.8147437752003887, + "pass@128": 0.8424242424242424 + }, + "16": { + "pass@1": 0.3059116809116809, + "pass@2": 0.39158751990936996, + "pass@4": 0.4727661555660242, + "pass@8": 0.5448737019577484, + "pass@16": 0.6040431432838912, + "pass@32": 0.6528764342680178, + "pass@64": 0.694521859735016, + "pass@128": 0.7293447293447294 + }, + "17": { + "pass@1": 0.26052989130434784, + "pass@2": 0.34320820291485293, + "pass@4": 0.42293633977041684, + "pass@8": 0.4927356348756277, + "pass@16": 0.5482837994495754, + "pass@32": 0.5921106992309274, + "pass@64": 0.6290502589670679, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.2153262867647059, + "pass@2": 0.28677049559981455, + "pass@4": 0.35674156300315396, + "pass@8": 0.4197868754911729, + "pass@16": 0.47482275196634255, + "pass@32": 0.5236327311772606, + "pass@64": 0.5672130434247589, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.16901506696428573, + "pass@2": 0.22576206294525675, + "pass@4": 0.28495160454273577, + "pass@8": 0.34010383127747634, + "pass@16": 0.38708927582856617, + "pass@32": 0.4276740630065036, + "pass@64": 0.4649332236307543, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.1504491017964072, + "pass@2": 0.2108255192371163, + "pass@4": 0.2735428047167756, + "pass@8": 0.3338204070406974, + "pass@16": 0.38676384570770866, + "pass@32": 0.4292259673414519, + "pass@64": 0.4663031080186876, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 575859, + "answer_accuracy": 0.7187886942802365, + "avg_response_len": 286.13477966727913, + "resp_tokens_sum": 229237451, + "pass_at_k": { + "pass@1": 0.7187886942802365, + "pass@2": 0.7548033784106732, + "pass@4": 0.7848112376220922, + "pass@8": 0.8098416329714997, + "pass@16": 0.8305795131153559, + "pass@32": 0.8481136410654446, + "pass@64": 0.8635051234293231, + "pass@128": 0.8771369228311232 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569496268656716, + "pass@2": 0.9678403014455282, + "pass@4": 0.976078659010907, + "pass@8": 0.9827045652363846, + "pass@16": 0.9880512471221918, + "pass@32": 0.9920857219167919, + "pass@64": 0.9938974306133376, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997487207602339, + "pass@2": 0.9999802142330892, + "pass@4": 0.9999999095507799, + "pass@8": 0.9999999999996626, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9922564338235295, + "pass@2": 0.994646320634553, + "pass@4": 0.9961220976973466, + "pass@8": 0.9970632553671733, + "pass@16": 0.997715072688465, + "pass@32": 0.9983497369764539, + "pass@64": 0.9992704955997272, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992360015772871, + "pass@2": 0.9998886118631859, + "pass@4": 0.9999968563669289, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998318829113924, + "pass@2": 0.9996340202332306, + "pass@4": 0.9999719820425929, + "pass@8": 0.9999998296794577, + "pass@16": 0.999999999997869, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995777027027027, + "pass@2": 0.9988557701402191, + "pass@4": 0.9998988651193375, + "pass@8": 0.9999992668226257, + "pass@16": 0.9999999999808709, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9933430466237942, + "pass@2": 0.9955922285236853, + "pass@4": 0.996922894083577, + "pass@8": 0.99825698853535, + "pass@16": 0.9992957181536803, + "pass@32": 0.9998834594758289, + "pass@64": 0.9999990174474539, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9458333333333333, + "pass@2": 0.9680735220597424, + "pass@4": 0.9784683795477946, + "pass@8": 0.9834253742459571, + "pass@16": 0.9859158447021265, + "pass@32": 0.9876978205871854, + "pass@64": 0.9894997112731746, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8640577217125383, + "pass@2": 0.9053041248284324, + "pass@4": 0.9285084223990344, + "pass@8": 0.9444538698747363, + "pass@16": 0.9565054855884125, + "pass@32": 0.9649179472227764, + "pass@64": 0.971280941814739, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7043306327160493, + "pass@2": 0.7786207713619127, + "pass@4": 0.8288944483791375, + "pass@8": 0.86676143621755, + "pass@16": 0.8943791026870926, + "pass@32": 0.9153670080570203, + "pass@64": 0.9331493513324162, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6179293397626113, + "pass@2": 0.703675538563985, + "pass@4": 0.7640525688183634, + "pass@8": 0.8070382454357383, + "pass@16": 0.8387920432900056, + "pass@32": 0.8638308703360127, + "pass@64": 0.8855514513931839, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.46923225308641975, + "pass@2": 0.5619528136239913, + "pass@4": 0.6342046055007012, + "pass@8": 0.6883368569053809, + "pass@16": 0.7319691872044063, + "pass@32": 0.7717383879708698, + "pass@64": 0.8095051745441575, + "pass@128": 0.8425925925925926 + }, + "16": { + "pass@1": 0.31752396755162243, + "pass@2": 0.3971402254198313, + "pass@4": 0.46753584065266157, + "pass@8": 0.5274963190047928, + "pass@16": 0.578633799879253, + "pass@32": 0.6236236229841198, + "pass@64": 0.6655990132285015, + "pass@128": 0.7079646017699115 + }, + "17": { + "pass@1": 0.2409403237951807, + "pass@2": 0.3118437084953988, + "pass@4": 0.38310006194632296, + "pass@8": 0.4517112008785566, + "pass@16": 0.5143629770086949, + "pass@32": 0.5692526943598468, + "pass@64": 0.6171244785989948, + "pass@128": 0.6566265060240963 + }, + "18": { + "pass@1": 0.21854838709677418, + "pass@2": 0.2914830454660909, + "pass@4": 0.36253027202244875, + "pass@8": 0.4232167175836581, + "pass@16": 0.4722504664411351, + "pass@32": 0.5141885209927322, + "pass@64": 0.5525587893884821, + "pass@128": 0.5870967741935483 + }, + "19": { + "pass@1": 0.1644873489425982, + "pass@2": 0.22967898815329338, + "pass@4": 0.296141610225459, + "pass@8": 0.35542356027993677, + "pass@16": 0.40438333117475245, + "pass@32": 0.44624875815883597, + "pass@64": 0.4841766947910102, + "pass@128": 0.5196374622356495 + }, + "20": { + "pass@1": 0.16366366366366367, + "pass@2": 0.22465292556335845, + "pass@4": 0.28830474653506144, + "pass@8": 0.3515397113540908, + "pass@16": 0.4098242301095972, + "pass@32": 0.4585545918829372, + "pass@64": 0.49711705635141545, + "pass@128": 0.5285285285285285 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ecfd54f3399104ed892d04d85c892bebb756d49b --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72b45f86223b6ed3a620ca6ab351bf510533278b22dd9c0708a9628915097190 +size 2587735856 diff --git a/results/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv b/results/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d644cddb939655f29848942311ea1fa7de64eb82 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op7-10,0.9928402777777777,0.4674421875,0.7163149671052632,0.0,0.0,0.0,200.1375373263889,332.59690703125,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch4-op7-10/summary.json b/results/composition/SDAR-100M-block16-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..96c13ac245b4a8519bb17faa1de18c7e7ed17f3e --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op7-10", + "id_acc": 0.9928402777777777, + "ood_acc": 0.4674421875, + "total_acc": 0.7163149671052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.1375373263889, + "ood_avg_resp_len": 332.59690703125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 576926, + "answer_accuracy": 0.7124935780904205, + "avg_response_len": 257.13824642349033, + "resp_tokens_sum": 208212038, + "pass_at_k": { + "pass@1": 0.7124935780904205, + "pass@2": 0.7482208665462601, + "pass@4": 0.7782977803705171, + "pass@8": 0.8033142972254913, + "pass@16": 0.8239431045015444, + "pass@32": 0.841313857869306, + "pass@64": 0.8566171919665101, + "pass@128": 0.8706923806512804 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639564896755162, + "pass@2": 0.9751933663159359, + "pass@4": 0.9805886808396739, + "pass@8": 0.9832729426842688, + "pass@16": 0.9853094435850058, + "pass@32": 0.9875159779599025, + "pass@64": 0.9903780082743677, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.997314453125, + "pass@2": 0.9981799028051181, + "pass@4": 0.9989848275801462, + "pass@8": 0.9996841121782726, + "pass@16": 0.9999734491629393, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9964285714285714, + "pass@2": 0.9987411729783777, + "pass@4": 0.9997791192767571, + "pass@8": 0.9999910052051663, + "pass@16": 0.999999987524699, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9973235124239709, + "pass@4": 0.9976157328200742, + "pass@8": 0.9981157895218378, + "pass@16": 0.998852517052828, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972725591715976, + "pass@2": 0.9980547919675722, + "pass@4": 0.9987735604794961, + "pass@8": 0.9995079830474547, + "pass@16": 0.999928825622786, + "pass@32": 0.9999991140832827, + "pass@64": 0.9999999999952738, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9991486378205128, + "pass@2": 0.9999641158136482, + "pass@4": 0.999999888836011, + "pass@8": 0.9999999999996302, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955325704225352, + "pass@2": 0.9969474326272597, + "pass@4": 0.9973978846834287, + "pass@8": 0.9976821039095084, + "pass@16": 0.998119293157022, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9345122167138811, + "pass@2": 0.9542368143695208, + "pass@4": 0.9657462063877991, + "pass@8": 0.9749512748470323, + "pass@16": 0.9827357531203197, + "pass@32": 0.9884344587547554, + "pass@64": 0.9925691216469926, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.8454493290960452, + "pass@2": 0.8801169285777846, + "pass@4": 0.8989873583069065, + "pass@8": 0.9130351649779792, + "pass@16": 0.9250958046726186, + "pass@32": 0.9364960935539219, + "pass@64": 0.9474650567648585, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.74047315830721, + "pass@2": 0.8119554217164869, + "pass@4": 0.8599147251969675, + "pass@8": 0.8945134207641785, + "pass@16": 0.9205661933194598, + "pass@32": 0.9404403427759488, + "pass@64": 0.9559823521026871, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6125632788161994, + "pass@2": 0.7013274370201388, + "pass@4": 0.7658476230190846, + "pass@8": 0.8097702298192307, + "pass@16": 0.8407102562368179, + "pass@32": 0.8644964958281782, + "pass@64": 0.8838021812976606, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.48457821531791906, + "pass@2": 0.5827305027081152, + "pass@4": 0.6590647869702699, + "pass@8": 0.7158310076759379, + "pass@16": 0.7589664128271321, + "pass@32": 0.7930825505983045, + "pass@64": 0.8204516163076292, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.2755040322580645, + "pass@2": 0.3586213328676657, + "pass@4": 0.43517604109970115, + "pass@8": 0.5017510566263869, + "pass@16": 0.5583572473734497, + "pass@32": 0.6078831915021241, + "pass@64": 0.6517453480905581, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.21780166184971098, + "pass@2": 0.29274348857584986, + "pass@4": 0.36758594012453627, + "pass@8": 0.4327260544001389, + "pass@16": 0.4852625887317101, + "pass@32": 0.5308733536032256, + "pass@64": 0.5731955827266514, + "pass@128": 0.615606936416185 + }, + "18": { + "pass@1": 0.18082589285714284, + "pass@2": 0.24475007030371193, + "pass@4": 0.30927755396646855, + "pass@8": 0.3685482161042458, + "pass@16": 0.4204588630253465, + "pass@32": 0.46585847061159913, + "pass@64": 0.5068792519157992, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.1753472222222222, + "pass@2": 0.23986560379276908, + "pass@4": 0.3042127763308865, + "pass@8": 0.3648899684541569, + "pass@16": 0.41793983675736257, + "pass@32": 0.46237397780199607, + "pass@64": 0.5029501083188042, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.2165424528859568, + "pass@4": 0.27821547475484476, + "pass@8": 0.3365076147164066, + "pass@16": 0.3888313889531125, + "pass@32": 0.4337362063462613, + "pass@64": 0.4738117632886543, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 589293, + "answer_accuracy": 0.71766976812159, + "avg_response_len": 266.5054985872954, + "resp_tokens_sum": 218832995, + "pass_at_k": { + "pass@1": 0.71766976812159, + "pass@2": 0.7541205482352454, + "pass@4": 0.7849935205585589, + "pass@8": 0.8106964627292033, + "pass@16": 0.831317315567593, + "pass@32": 0.8479933921096046, + "pass@64": 0.8623722337450352, + "pass@128": 0.875448168355417 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412384969325154, + "pass@2": 0.9570337030819772, + "pass@4": 0.9684386097635029, + "pass@8": 0.9755421990555402, + "pass@16": 0.9790090135040668, + "pass@32": 0.9808182150392867, + "pass@64": 0.9815547734367644, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963768115942029, + "pass@2": 0.9971339010612804, + "pass@4": 0.9974446123582379, + "pass@8": 0.9977680501821345, + "pass@16": 0.9983124881128967, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999841363400491, + "pass@4": 0.9999999435958757, + "pass@8": 0.9999999999999097, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997674313583815, + "pass@2": 0.998275069978608, + "pass@4": 0.9989801556597332, + "pass@8": 0.9996539139236084, + "pass@16": 0.9999649956979594, + "pass@32": 0.9999998098093686, + "pass@64": 0.9999999999999141, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957306338028169, + "pass@2": 0.9968081124542529, + "pass@4": 0.9971466248937193, + "pass@8": 0.9971827493473799, + "pass@16": 0.9971830985789736, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.993310254491018, + "pass@2": 0.9940558106228489, + "pass@4": 0.994196379457059, + "pass@8": 0.9943759135003243, + "pass@16": 0.9947162761091989, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9478068524096386, + "pass@2": 0.9611561403092685, + "pass@4": 0.9686419691702693, + "pass@8": 0.9750669382908692, + "pass@16": 0.9806081356024587, + "pass@32": 0.9842428892059527, + "pass@64": 0.9862549466314855, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8410315438871473, + "pass@2": 0.8755083244884359, + "pass@4": 0.8937380000305601, + "pass@8": 0.905965360565369, + "pass@16": 0.9151980908995769, + "pass@32": 0.9235809759059825, + "pass@64": 0.9339121914379174, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7262123599439776, + "pass@2": 0.8062757091025381, + "pass@4": 0.8566690912585506, + "pass@8": 0.8882637919932588, + "pass@16": 0.9082264797954909, + "pass@32": 0.9222958616323473, + "pass@64": 0.9343518871725501, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6329952485380117, + "pass@2": 0.7221707792282543, + "pass@4": 0.7828848704767167, + "pass@8": 0.8258118272832482, + "pass@16": 0.8579023706600551, + "pass@32": 0.8832258032244631, + "pass@64": 0.9049950902372479, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.4446022727272727, + "pass@2": 0.5437966028394173, + "pass@4": 0.6285262524002679, + "pass@8": 0.6960194300230498, + "pass@16": 0.7475932370874276, + "pass@32": 0.785985388651888, + "pass@64": 0.8147437752003887, + "pass@128": 0.8424242424242424 + }, + "16": { + "pass@1": 0.3059116809116809, + "pass@2": 0.39158751990936996, + "pass@4": 0.4727661555660242, + "pass@8": 0.5448737019577484, + "pass@16": 0.6040431432838912, + "pass@32": 0.6528764342680178, + "pass@64": 0.694521859735016, + "pass@128": 0.7293447293447294 + }, + "17": { + "pass@1": 0.26052989130434784, + "pass@2": 0.34320820291485293, + "pass@4": 0.42293633977041684, + "pass@8": 0.4927356348756277, + "pass@16": 0.5482837994495754, + "pass@32": 0.5921106992309274, + "pass@64": 0.6290502589670679, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.2153262867647059, + "pass@2": 0.28677049559981455, + "pass@4": 0.35674156300315396, + "pass@8": 0.4197868754911729, + "pass@16": 0.47482275196634255, + "pass@32": 0.5236327311772606, + "pass@64": 0.5672130434247589, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.16901506696428573, + "pass@2": 0.22576206294525675, + "pass@4": 0.28495160454273577, + "pass@8": 0.34010383127747634, + "pass@16": 0.38708927582856617, + "pass@32": 0.4276740630065036, + "pass@64": 0.4649332236307543, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.1504491017964072, + "pass@2": 0.2108255192371163, + "pass@4": 0.2735428047167756, + "pass@8": 0.3338204070406974, + "pass@16": 0.38676384570770866, + "pass@32": 0.4292259673414519, + "pass@64": 0.4663031080186876, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 575859, + "answer_accuracy": 0.7187886942802365, + "avg_response_len": 286.13477966727913, + "resp_tokens_sum": 229237451, + "pass_at_k": { + "pass@1": 0.7187886942802365, + "pass@2": 0.7548033784106732, + "pass@4": 0.7848112376220922, + "pass@8": 0.8098416329714997, + "pass@16": 0.8305795131153559, + "pass@32": 0.8481136410654446, + "pass@64": 0.8635051234293231, + "pass@128": 0.8771369228311232 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569496268656716, + "pass@2": 0.9678403014455282, + "pass@4": 0.976078659010907, + "pass@8": 0.9827045652363846, + "pass@16": 0.9880512471221918, + "pass@32": 0.9920857219167919, + "pass@64": 0.9938974306133376, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997487207602339, + "pass@2": 0.9999802142330892, + "pass@4": 0.9999999095507799, + "pass@8": 0.9999999999996626, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9922564338235295, + "pass@2": 0.994646320634553, + "pass@4": 0.9961220976973466, + "pass@8": 0.9970632553671733, + "pass@16": 0.997715072688465, + "pass@32": 0.9983497369764539, + "pass@64": 0.9992704955997272, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992360015772871, + "pass@2": 0.9998886118631859, + "pass@4": 0.9999968563669289, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998318829113924, + "pass@2": 0.9996340202332306, + "pass@4": 0.9999719820425929, + "pass@8": 0.9999998296794577, + "pass@16": 0.999999999997869, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995777027027027, + "pass@2": 0.9988557701402191, + "pass@4": 0.9998988651193375, + "pass@8": 0.9999992668226257, + "pass@16": 0.9999999999808709, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9933430466237942, + "pass@2": 0.9955922285236853, + "pass@4": 0.996922894083577, + "pass@8": 0.99825698853535, + "pass@16": 0.9992957181536803, + "pass@32": 0.9998834594758289, + "pass@64": 0.9999990174474539, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9458333333333333, + "pass@2": 0.9680735220597424, + "pass@4": 0.9784683795477946, + "pass@8": 0.9834253742459571, + "pass@16": 0.9859158447021265, + "pass@32": 0.9876978205871854, + "pass@64": 0.9894997112731746, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8640577217125383, + "pass@2": 0.9053041248284324, + "pass@4": 0.9285084223990344, + "pass@8": 0.9444538698747363, + "pass@16": 0.9565054855884125, + "pass@32": 0.9649179472227764, + "pass@64": 0.971280941814739, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7043306327160493, + "pass@2": 0.7786207713619127, + "pass@4": 0.8288944483791375, + "pass@8": 0.86676143621755, + "pass@16": 0.8943791026870926, + "pass@32": 0.9153670080570203, + "pass@64": 0.9331493513324162, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6179293397626113, + "pass@2": 0.703675538563985, + "pass@4": 0.7640525688183634, + "pass@8": 0.8070382454357383, + "pass@16": 0.8387920432900056, + "pass@32": 0.8638308703360127, + "pass@64": 0.8855514513931839, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.46923225308641975, + "pass@2": 0.5619528136239913, + "pass@4": 0.6342046055007012, + "pass@8": 0.6883368569053809, + "pass@16": 0.7319691872044063, + "pass@32": 0.7717383879708698, + "pass@64": 0.8095051745441575, + "pass@128": 0.8425925925925926 + }, + "16": { + "pass@1": 0.31752396755162243, + "pass@2": 0.3971402254198313, + "pass@4": 0.46753584065266157, + "pass@8": 0.5274963190047928, + "pass@16": 0.578633799879253, + "pass@32": 0.6236236229841198, + "pass@64": 0.6655990132285015, + "pass@128": 0.7079646017699115 + }, + "17": { + "pass@1": 0.2409403237951807, + "pass@2": 0.3118437084953988, + "pass@4": 0.38310006194632296, + "pass@8": 0.4517112008785566, + "pass@16": 0.5143629770086949, + "pass@32": 0.5692526943598468, + "pass@64": 0.6171244785989948, + "pass@128": 0.6566265060240963 + }, + "18": { + "pass@1": 0.21854838709677418, + "pass@2": 0.2914830454660909, + "pass@4": 0.36253027202244875, + "pass@8": 0.4232167175836581, + "pass@16": 0.4722504664411351, + "pass@32": 0.5141885209927322, + "pass@64": 0.5525587893884821, + "pass@128": 0.5870967741935483 + }, + "19": { + "pass@1": 0.1644873489425982, + "pass@2": 0.22967898815329338, + "pass@4": 0.296141610225459, + "pass@8": 0.35542356027993677, + "pass@16": 0.40438333117475245, + "pass@32": 0.44624875815883597, + "pass@64": 0.4841766947910102, + "pass@128": 0.5196374622356495 + }, + "20": { + "pass@1": 0.16366366366366367, + "pass@2": 0.22465292556335845, + "pass@4": 0.28830474653506144, + "pass@8": 0.3515397113540908, + "pass@16": 0.4098242301095972, + "pass@32": 0.4585545918829372, + "pass@64": 0.49711705635141545, + "pass@128": 0.5285285285285285 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f77253f9c5f2043c0596b56b3f0560349689c5ff --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fd488450c44bfa545a35cc390d60b8190a3a29317b9041b92c1dd21c8dbe957 +size 1789580934 diff --git a/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json b/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0c97cf28b5bceeeaf3cfe8aab18778f2e6941e36 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9921467013888889, + "avg_response_len": 199.46203125, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9921467013888889, + "pass@2": 0.9944876284995613, + "pass@4": 0.9959433585697617, + "pass@8": 0.9968784496129127, + "pass@16": 0.9976004460052793, + "pass@32": 0.9981871074093316, + "pass@64": 0.9986349505024621, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9524140625, + "pass@2": 0.9645937500000002, + "pass@4": 0.9730021943194608, + "pass@8": 0.9786367820253195, + "pass@16": 0.9831046505061921, + "pass@32": 0.9868706993361868, + "pass@64": 0.9899770280239669, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9987109375, + "pass@2": 0.999171998031496, + "pass@4": 0.9994346713535808, + "pass@8": 0.9996911656406087, + "pass@16": 0.9999126144227602, + "pass@32": 0.9999949322702752, + "pass@64": 0.9999999977569969, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.993890625, + "pass@2": 0.996357775590551, + "pass@4": 0.9976947060367454, + "pass@8": 0.9983995073818661, + "pass@16": 0.9989712933683291, + "pass@32": 0.9994882787743641, + "pass@64": 0.9998748568874752, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.997625, + "pass@2": 0.9985620078740157, + "pass@4": 0.9990585259655044, + "pass@8": 0.9993178674757099, + "pass@16": 0.9995588461117559, + "pass@32": 0.9998290717621782, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997484375, + "pass@2": 0.9984462352362204, + "pass@4": 0.9990977215973003, + "pass@8": 0.9995759792654791, + "pass@16": 0.9998778554116803, + "pass@32": 0.9999900984417422, + "pass@64": 0.999999987930507, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995140625, + "pass@2": 0.9973559301181102, + "pass@4": 0.9985708786089239, + "pass@8": 0.9991065163275745, + "pass@16": 0.9993309358545857, + "pass@32": 0.9995814582857677, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9940546875, + "pass@2": 0.9959009596456694, + "pass@4": 0.9966315292463443, + "pass@8": 0.99717822839966, + "pass@16": 0.9976478183722098, + "pass@32": 0.9979294278134688, + "pass@64": 0.9979985548661734, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9524140625, + "2": 1.0, + "3": 1.0, + "4": 0.9987109375, + "5": 0.993890625, + "6": 0.997625, + "7": 0.997484375, + "8": 0.995140625, + "9": 0.9940546875 + }, + "per_op_avg_response_len": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376348, + "answer_accuracy": 0.9929816784869976, + "avg_response_len": 189.93894587977036, + "resp_tokens_sum": 71988380, + "pass_at_k": { + "pass@1": 0.9929816784869976, + "pass@2": 0.9952404553154262, + "pass@4": 0.996518110166574, + "pass@8": 0.9972523121377939, + "pass@16": 0.9978434752201003, + "pass@32": 0.9984079378454725, + "pass@64": 0.9988835021935919, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9574575958702065, + "pass@2": 0.9697600196269713, + "pass@4": 0.9770701256369501, + "pass@8": 0.9807352442070735, + "pass@16": 0.9835831385400656, + "pass@32": 0.9868139186902647, + "pass@64": 0.9902929813385202, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996923828125, + "pass@2": 0.9976178026574803, + "pass@4": 0.9982453347628422, + "pass@8": 0.9990349229744625, + "pass@16": 0.9997269200711651, + "pass@32": 0.9999841633446103, + "pass@64": 0.9999999929906153, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9981666354205724, + "pass@4": 0.9995852298224628, + "pass@8": 0.9999710725449563, + "pass@16": 0.9999998592866416, + "pass@32": 0.9999999999993872, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968890344311377, + "pass@2": 0.997264575180348, + "pass@4": 0.997534851556729, + "pass@8": 0.9979835642250304, + "pass@16": 0.9986793120796699, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973187869822485, + "pass@2": 0.9984067785258351, + "pass@4": 0.9992237471795316, + "pass@8": 0.9998073811974663, + "pass@16": 0.9999900064095892, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983974358974359, + "pass@2": 0.999743290051484, + "pass@4": 0.9999842372467864, + "pass@8": 0.9999999472370541, + "pass@16": 0.9999999999998607, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972687007874016, + "pass@4": 0.997876473539399, + "pass@8": 0.9984546377928358, + "pass@16": 0.999188066655105, + "pass@32": 0.9998084093975909, + "pass@64": 0.9999959303192606, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387303, + "answer_accuracy": 0.991417001146789, + "avg_response_len": 197.48486648099606, + "resp_tokens_sum": 77148648, + "pass_at_k": { + "pass@1": 0.991417001146789, + "pass@2": 0.993621147485578, + "pass@4": 0.9950336770751789, + "pass@8": 0.9959590200409408, + "pass@16": 0.9966420865580624, + "pass@32": 0.9971659138581003, + "pass@64": 0.9976167577265961, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421731211656442, + "pass@2": 0.9570635174629243, + "pass@4": 0.9675180118397778, + "pass@8": 0.9744761195433342, + "pass@16": 0.9791587744555608, + "pass@32": 0.9822004591222739, + "pass@64": 0.984571936009979, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956295289855073, + "pass@2": 0.9968182985279016, + "pass@4": 0.9973139007080636, + "pass@8": 0.9976146824957167, + "pass@16": 0.998064779875966, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9972242120343839, + "pass@2": 0.9987002374613632, + "pass@4": 0.9996714137452016, + "pass@8": 0.9999752561101265, + "pass@16": 0.9999998735735035, + "pass@32": 0.999999999999447, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973807803468208, + "pass@2": 0.9977498748350099, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954665492957746, + "pass@2": 0.9968725740268383, + "pass@4": 0.9974011850631346, + "pass@8": 0.9976822657434412, + "pass@16": 0.9981192931743675, + "pass@32": 0.9988210100498002, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930295658682635, + "pass@2": 0.9939206233202886, + "pass@4": 0.9940102609741148, + "pass@8": 0.9940119756217468, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379302, + "answer_accuracy": 0.9920645714763977, + "avg_response_len": 210.92241379310346, + "resp_tokens_sum": 80643232, + "pass_at_k": { + "pass@1": 0.9920645714763977, + "pass@2": 0.9946266910549394, + "pass@4": 0.9963030868735149, + "pass@8": 0.9974472785775886, + "pass@16": 0.9983387464832911, + "pass@32": 0.9990116148739929, + "pass@64": 0.999428911933499, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9572761194029851, + "pass@2": 0.9666937213538606, + "pass@4": 0.9742225369216908, + "pass@8": 0.9805621470686411, + "pass@16": 0.9864603166226772, + "pass@32": 0.991472929033808, + "pass@64": 0.9949171827192708, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998089774140073, + "pass@4": 0.9999887842966998, + "pass@8": 0.9999999716046218, + "pass@16": 0.9999999999999635, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906479779411764, + "pass@2": 0.9942146248263086, + "pass@4": 0.996329596759964, + "pass@8": 0.9977398649093597, + "pass@16": 0.9989382018700852, + "pass@32": 0.9997259417849353, + "pass@64": 0.9999908945046169, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9988416798107256, + "pass@2": 0.9997768356143968, + "pass@4": 0.9999891550425282, + "pass@8": 0.9999999767890079, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997774920886076, + "pass@2": 0.9992509094986542, + "pass@4": 0.9998849730413445, + "pass@8": 0.9999968489460502, + "pass@16": 0.9999999985879974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9917417417417418, + "pass@2": 0.9956344139415007, + "pass@4": 0.998493621292834, + "pass@8": 0.999787773125201, + "pass@16": 0.9999957560892757, + "pass@32": 0.9999999991534195, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932174437299035, + "pass@2": 0.9964665037850975, + "pass@4": 0.9980255754807175, + "pass@8": 0.9991216467059164, + "pass@16": 0.9997943881339146, + "pass@32": 0.9999917764544182, + "pass@64": 0.9999999987229452, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5692328125, + "avg_response_len": 350.66635078125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5692328125, + "pass@2": 0.638573991141724, + "pass@4": 0.6956425176509109, + "pass@8": 0.7415618754091828, + "pass@16": 0.7774673995948668, + "pass@32": 0.805792751873983, + "pass@64": 0.8294758529319611, + "pass@128": 0.8504 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9613984375, + "pass@2": 0.9735031988188977, + "pass@4": 0.9804231168916386, + "pass@8": 0.9847109635922205, + "pass@16": 0.9870854668143153, + "pass@32": 0.9886758259692309, + "pass@64": 0.9904900450431707, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.899875, + "pass@2": 0.9206909448818899, + "pass@4": 0.9333334387889014, + "pass@8": 0.9429585488390639, + "pass@16": 0.9508381683007954, + "pass@32": 0.957124237853498, + "pass@64": 0.9620219505951612, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.8238984375, + "pass@2": 0.8724411909448823, + "pass@4": 0.9021637046306712, + "pass@8": 0.9231090688142323, + "pass@16": 0.9379672291934675, + "pass@32": 0.9484869565082493, + "pass@64": 0.9568897147822485, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.7794453125, + "pass@2": 0.8445839074803148, + "pass@4": 0.8855294000749906, + "pass@8": 0.9114116915660494, + "pass@16": 0.9273511399284904, + "pass@32": 0.9386215902003806, + "pass@64": 0.9495939611176091, + "pass@128": 0.962 + }, + "15": { + "pass@1": 0.6651171875, + "pass@2": 0.7486331200787407, + "pass@4": 0.806329887701538, + "pass@8": 0.8473808946123879, + "pass@16": 0.8765889720497365, + "pass@32": 0.8979515241106913, + "pass@64": 0.9145665425705841, + "pass@128": 0.928 + }, + "16": { + "pass@1": 0.4879609375, + "pass@2": 0.5945366633858277, + "pass@4": 0.6776242561867271, + "pass@8": 0.739991947562845, + "pass@16": 0.7879800006920792, + "pass@32": 0.8266620166339039, + "pass@64": 0.8592523951014696, + "pass@128": 0.89 + }, + "17": { + "pass@1": 0.369609375, + "pass@2": 0.47383956692913415, + "pass@4": 0.566708111173604, + "pass@8": 0.6421709788558302, + "pass@16": 0.699630230918739, + "pass@32": 0.7440501332532892, + "pass@64": 0.7798027890890089, + "pass@128": 0.811 + }, + "18": { + "pass@1": 0.3044921875, + "pass@2": 0.402151697834646, + "pass@4": 0.49101950562429714, + "pass@8": 0.5642945200089501, + "pass@16": 0.6220010627856662, + "pass@32": 0.6671294229443815, + "pass@64": 0.7039513297789204, + "pass@128": 0.734 + }, + "19": { + "pass@1": 0.20428125, + "pass@2": 0.28356422244094465, + "pass@4": 0.363903641076115, + "pass@8": 0.4365091889268906, + "pass@16": 0.4967211171125234, + "pass@32": 0.5470671625654399, + "pass@64": 0.5925308099061153, + "pass@128": 0.635 + }, + "20": { + "pass@1": 0.19625, + "pass@2": 0.27179539862204705, + "pass@4": 0.3493901143607049, + "pass@8": 0.4230809513134006, + "pass@16": 0.4885106081527199, + "pass@32": 0.5421586487007338, + "pass@64": 0.5856589913352467, + "pass@128": 0.622 + } + }, + "per_op_accuracy": { + "11": 0.9613984375, + "12": 0.899875, + "13": 0.8238984375, + "14": 0.7794453125, + "15": 0.6651171875, + "16": 0.4879609375, + "17": 0.369609375, + "18": 0.3044921875, + "19": 0.20428125, + "20": 0.19625 + }, + "per_op_avg_response_len": { + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 245625, + "answer_accuracy": 0.5702660661218425, + "avg_response_len": 332.6203403603269, + "resp_tokens_sum": 143266233, + "pass_at_k": { + "pass@1": 0.5702660661218425, + "pass@2": 0.6388949541364937, + "pass@4": 0.6949541250137222, + "pass@8": 0.7397971193299135, + "pass@16": 0.7748348215257485, + "pass@32": 0.8027802603018277, + "pass@64": 0.8269381943859584, + "pass@128": 0.8484398216939079 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9572193696883853, + "pass@2": 0.9719146628449066, + "pass@4": 0.981724504514839, + "pass@8": 0.9890359200866053, + "pass@16": 0.9932697779011666, + "pass@32": 0.9957361168730702, + "pass@64": 0.9978415915007174, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.8983271539548022, + "pass@2": 0.920239098158281, + "pass@4": 0.9334048053315367, + "pass@8": 0.9434168617466765, + "pass@16": 0.9517176538857137, + "pass@32": 0.9580437536392892, + "pass@64": 0.961533417356335, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8390722962382445, + "pass@2": 0.8873212765038385, + "pass@4": 0.9195430402704361, + "pass@8": 0.9430086769866958, + "pass@16": 0.9574399410309512, + "pass@32": 0.9647881464197946, + "pass@64": 0.9690691600729979, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7840975467289719, + "pass@2": 0.8466142192214293, + "pass@4": 0.8872996447873914, + "pass@8": 0.9131255212050312, + "pass@16": 0.9296828208789675, + "pass@32": 0.9427319318749825, + "pass@64": 0.9554954405969336, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.693099710982659, + "pass@2": 0.7778968583587461, + "pass@4": 0.833728548292012, + "pass@8": 0.8711126510602295, + "pass@16": 0.8958576584940476, + "pass@32": 0.9133306283181201, + "pass@64": 0.9284685661529859, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.4913054435483871, + "pass@2": 0.5962447612395223, + "pass@4": 0.6747601975156323, + "pass@8": 0.7348407821560055, + "pass@16": 0.7834148610294204, + "pass@32": 0.8244895329338998, + "pass@64": 0.8601964216978978, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.35248825867052025, + "pass@2": 0.4572849580128351, + "pass@4": 0.5492719623919837, + "pass@8": 0.6229445136286828, + "pass@16": 0.6790291809157839, + "pass@32": 0.7227658365318381, + "pass@64": 0.7602209527875393, + "pass@128": 0.7976878612716763 + }, + "18": { + "pass@1": 0.27520089285714283, + "pass@2": 0.3652481721034869, + "pass@4": 0.4477995007766887, + "pass@8": 0.5185924991538704, + "pass@16": 0.5777052275710203, + "pass@32": 0.6248875096332369, + "pass@64": 0.6638893307765378, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.2045795795795796, + "pass@2": 0.28665747440353734, + "pass@4": 0.368495601338121, + "pass@8": 0.44158292317157055, + "pass@16": 0.5021246498473418, + "pass@32": 0.5528371750040144, + "pass@64": 0.5987933372999518, + "pass@128": 0.6396396396396397 + }, + "20": { + "pass@1": 0.19566441441441443, + "pass@2": 0.2724017078101725, + "pass@4": 0.34911851418347484, + "pass@8": 0.41767593951939913, + "pass@16": 0.47676148728079454, + "pass@32": 0.5280550430504534, + "pass@64": 0.5749912993919669, + "pass@128": 0.6156156156156156 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 248292, + "answer_accuracy": 0.5768008474576272, + "avg_response_len": 346.25764523862625, + "resp_tokens_sum": 149051451, + "pass_at_k": { + "pass@1": 0.5768008474576272, + "pass@2": 0.6462570475133562, + "pass@4": 0.7029366738948009, + "pass@8": 0.7483487965631102, + "pass@16": 0.7839769463433715, + "pass@32": 0.8124999007618324, + "pass@64": 0.8363338575179757, + "pass@128": 0.8581623550401427 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9631024096385542, + "pass@2": 0.972274185323973, + "pass@4": 0.9774970881840069, + "pass@8": 0.9805179334532464, + "pass@16": 0.9822529509986989, + "pass@32": 0.9834096807350399, + "pass@64": 0.9849397431912068, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8928781347962382, + "pass@2": 0.9105734579024021, + "pass@4": 0.920715678859108, + "pass@8": 0.928907449166582, + "pass@16": 0.9358474938634546, + "pass@32": 0.9417129174503823, + "pass@64": 0.9473011819327243, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8309042366946778, + "pass@2": 0.8788773875692008, + "pass@4": 0.9060005905249233, + "pass@8": 0.9230680510501225, + "pass@16": 0.9352975407775181, + "pass@32": 0.9462738870529341, + "pass@64": 0.956931111957274, + "pass@128": 0.9663865546218487 + }, + "14": { + "pass@1": 0.7888797514619883, + "pass@2": 0.8581284966846247, + "pass@4": 0.9010667306937505, + "pass@8": 0.9274503748185816, + "pass@16": 0.9419725164541798, + "pass@32": 0.9505580533617753, + "pass@64": 0.9588922804756361, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.6662642045454545, + "pass@2": 0.7540548198520639, + "pass@4": 0.8157365976980145, + "pass@8": 0.8565477326282641, + "pass@16": 0.8831288202183204, + "pass@32": 0.9034765538582886, + "pass@64": 0.9217986009661605, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.5013799857549858, + "pass@2": 0.6062000162639922, + "pass@4": 0.688734711846916, + "pass@8": 0.7507240841046224, + "pass@16": 0.7988058883708389, + "pass@32": 0.8382289811645698, + "pass@64": 0.8684671985838386, + "pass@128": 0.8945868945868946 + }, + "17": { + "pass@1": 0.39540469720496896, + "pass@2": 0.5015440070670513, + "pass@4": 0.5931639159429605, + "pass@8": 0.6665612968773015, + "pass@16": 0.7219907477028704, + "pass@32": 0.7646644496183821, + "pass@64": 0.7996400240328937, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.32417279411764705, + "pass@2": 0.42494535954145407, + "pass@4": 0.517787185425351, + "pass@8": 0.5978804895419353, + "pass@16": 0.6618267341101194, + "pass@32": 0.7122069429681127, + "pass@64": 0.7544866903270767, + "pass@128": 0.788235294117647 + }, + "19": { + "pass@1": 0.20305524553571427, + "pass@2": 0.2782470794666291, + "pass@4": 0.35577190797578867, + "pass@8": 0.42750031369822505, + "pass@16": 0.4901520073095929, + "pass@32": 0.5430748566756388, + "pass@64": 0.5864859314590565, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.20106661676646706, + "pass@2": 0.2747646930548353, + "pass@4": 0.3491225167774687, + "pass@8": 0.42036866918817706, + "pass@16": 0.4844573511671379, + "pass@32": 0.5371766445282311, + "pass@64": 0.5800651608494923, + "pass@128": 0.6197604790419161 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 234701, + "answer_accuracy": 0.5603916755806846, + "avg_response_len": 373.75660194452934, + "resp_tokens_sum": 156535245, + "pass_at_k": { + "pass@1": 0.5603916755806846, + "pass@2": 0.6303471699146198, + "pass@4": 0.6888534570690135, + "pass@8": 0.7364011139074038, + "pass@16": 0.7734842148415902, + "pass@32": 0.8019971811008832, + "pass@64": 0.825036901704703, + "pass@128": 0.844437652811736 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9642857142857143, + "pass@2": 0.9765787089113863, + "pass@4": 0.9820486778438408, + "pass@8": 0.9842835869687955, + "pass@16": 0.9852484301067794, + "pass@32": 0.9863141673301701, + "pass@64": 0.9881014873140843, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9083763379204893, + "pass@2": 0.9310500644128201, + "pass@4": 0.9455652481513205, + "pass@8": 0.956169735463612, + "pass@16": 0.9645099947486562, + "pass@32": 0.9711630837875163, + "pass@64": 0.9769114489739431, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8012393904320988, + "pass@2": 0.8506990011665209, + "pass@4": 0.8808248887870499, + "pass@8": 0.9035617488598224, + "pass@16": 0.9217366232932731, + "pass@32": 0.9348758028470283, + "pass@64": 0.9448526103401694, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7654395400593472, + "pass@2": 0.8289044428608145, + "pass@4": 0.8680753477773732, + "pass@8": 0.8935025847812458, + "pass@16": 0.9102918510356471, + "pass@32": 0.9225928362575211, + "pass@64": 0.934536393956455, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6340663580246914, + "pass@2": 0.7118602362204722, + "pass@4": 0.7674899159017153, + "pass@8": 0.8127011283278409, + "pass@16": 0.849350961539353, + "pass@32": 0.8759008146277355, + "pass@64": 0.892354569021042, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.4710084808259587, + "pass@2": 0.580898471070541, + "pass@4": 0.668739560762868, + "pass@8": 0.7335904176217118, + "pass@16": 0.7809455072412861, + "pass@32": 0.8166722390431593, + "pass@64": 0.8488481347262333, + "pass@128": 0.8849557522123894 + }, + "17": { + "pass@1": 0.36243411144578314, + "pass@2": 0.4642222927141631, + "pass@4": 0.5592205760612748, + "pass@8": 0.6385525287524542, + "pass@16": 0.6994129926552873, + "pass@32": 0.7462385573378134, + "pass@64": 0.7809706376082151, + "pass@128": 0.8042168674698795 + }, + "18": { + "pass@1": 0.31597782258064516, + "pass@2": 0.4188174688849377, + "pass@4": 0.5104581848639886, + "pass@8": 0.5790573511639918, + "pass@16": 0.6283327210915123, + "pass@32": 0.6653820450438427, + "pass@64": 0.6937567396642746, + "pass@128": 0.7161290322580646 + }, + "19": { + "pass@1": 0.2052256419939577, + "pass@2": 0.28584974248876, + "pass@4": 0.36753850377841757, + "pass@8": 0.44054975863490586, + "pass@16": 0.49795327570192016, + "pass@32": 0.5453148986286661, + "pass@64": 0.5923666332779112, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.19200450450450451, + "pass@2": 0.26821087820103573, + "pass@4": 0.3499301157175173, + "pass@8": 0.4312063902239929, + "pass@16": 0.5043251579531292, + "pass@32": 0.5612592194970087, + "pass@64": 0.6019373120540266, + "pass@128": 0.6306306306306306 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7695604440789474, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7695604440789474, + "pass@2": 0.8071646614691095, + "pass@4": 0.8378902844019319, + "pass@8": 0.8625013052951703, + "pass@16": 0.8817409478945274, + "pass@32": 0.8969269202854718, + "pass@64": 0.9096038465179878, + "pass@128": 0.9207894736842105 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9524140625, + "pass@2": 0.9645937500000002, + "pass@4": 0.9730021943194608, + "pass@8": 0.9786367820253195, + "pass@16": 0.9831046505061921, + "pass@32": 0.9868706993361868, + "pass@64": 0.9899770280239669, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9987109375, + "pass@2": 0.999171998031496, + "pass@4": 0.9994346713535808, + "pass@8": 0.9996911656406087, + "pass@16": 0.9999126144227602, + "pass@32": 0.9999949322702752, + "pass@64": 0.9999999977569969, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.993890625, + "pass@2": 0.996357775590551, + "pass@4": 0.9976947060367454, + "pass@8": 0.9983995073818661, + "pass@16": 0.9989712933683291, + "pass@32": 0.9994882787743641, + "pass@64": 0.9998748568874752, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.997625, + "pass@2": 0.9985620078740157, + "pass@4": 0.9990585259655044, + "pass@8": 0.9993178674757099, + "pass@16": 0.9995588461117559, + "pass@32": 0.9998290717621782, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997484375, + "pass@2": 0.9984462352362204, + "pass@4": 0.9990977215973003, + "pass@8": 0.9995759792654791, + "pass@16": 0.9998778554116803, + "pass@32": 0.9999900984417422, + "pass@64": 0.999999987930507, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995140625, + "pass@2": 0.9973559301181102, + "pass@4": 0.9985708786089239, + "pass@8": 0.9991065163275745, + "pass@16": 0.9993309358545857, + "pass@32": 0.9995814582857677, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9940546875, + "pass@2": 0.9959009596456694, + "pass@4": 0.9966315292463443, + "pass@8": 0.99717822839966, + "pass@16": 0.9976478183722098, + "pass@32": 0.9979294278134688, + "pass@64": 0.9979985548661734, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.9613984375, + "pass@2": 0.9735031988188977, + "pass@4": 0.9804231168916386, + "pass@8": 0.9847109635922205, + "pass@16": 0.9870854668143153, + "pass@32": 0.9886758259692309, + "pass@64": 0.9904900450431707, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.899875, + "pass@2": 0.9206909448818899, + "pass@4": 0.9333334387889014, + "pass@8": 0.9429585488390639, + "pass@16": 0.9508381683007954, + "pass@32": 0.957124237853498, + "pass@64": 0.9620219505951612, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.8238984375, + "pass@2": 0.8724411909448823, + "pass@4": 0.9021637046306712, + "pass@8": 0.9231090688142323, + "pass@16": 0.9379672291934675, + "pass@32": 0.9484869565082493, + "pass@64": 0.9568897147822485, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.7794453125, + "pass@2": 0.8445839074803148, + "pass@4": 0.8855294000749906, + "pass@8": 0.9114116915660494, + "pass@16": 0.9273511399284904, + "pass@32": 0.9386215902003806, + "pass@64": 0.9495939611176091, + "pass@128": 0.962 + }, + "15": { + "pass@1": 0.6651171875, + "pass@2": 0.7486331200787407, + "pass@4": 0.806329887701538, + "pass@8": 0.8473808946123879, + "pass@16": 0.8765889720497365, + "pass@32": 0.8979515241106913, + "pass@64": 0.9145665425705841, + "pass@128": 0.928 + }, + "16": { + "pass@1": 0.4879609375, + "pass@2": 0.5945366633858277, + "pass@4": 0.6776242561867271, + "pass@8": 0.739991947562845, + "pass@16": 0.7879800006920792, + "pass@32": 0.8266620166339039, + "pass@64": 0.8592523951014696, + "pass@128": 0.89 + }, + "17": { + "pass@1": 0.369609375, + "pass@2": 0.47383956692913415, + "pass@4": 0.566708111173604, + "pass@8": 0.6421709788558302, + "pass@16": 0.699630230918739, + "pass@32": 0.7440501332532892, + "pass@64": 0.7798027890890089, + "pass@128": 0.811 + }, + "18": { + "pass@1": 0.3044921875, + "pass@2": 0.402151697834646, + "pass@4": 0.49101950562429714, + "pass@8": 0.5642945200089501, + "pass@16": 0.6220010627856662, + "pass@32": 0.6671294229443815, + "pass@64": 0.7039513297789204, + "pass@128": 0.734 + }, + "19": { + "pass@1": 0.20428125, + "pass@2": 0.28356422244094465, + "pass@4": 0.363903641076115, + "pass@8": 0.4365091889268906, + "pass@16": 0.4967211171125234, + "pass@32": 0.5470671625654399, + "pass@64": 0.5925308099061153, + "pass@128": 0.635 + }, + "20": { + "pass@1": 0.19625, + "pass@2": 0.27179539862204705, + "pass@4": 0.3493901143607049, + "pass@8": 0.4230809513134006, + "pass@16": 0.4885106081527199, + "pass@32": 0.5421586487007338, + "pass@64": 0.5856589913352467, + "pass@128": 0.622 + } + }, + "per_op_accuracy": { + "10": 0.9524140625, + "2": 1.0, + "3": 1.0, + "4": 0.9987109375, + "5": 0.993890625, + "6": 0.997625, + "7": 0.997484375, + "8": 0.995140625, + "9": 0.9940546875, + "11": 0.9613984375, + "12": 0.899875, + "13": 0.8238984375, + "14": 0.7794453125, + "15": 0.6651171875, + "16": 0.4879609375, + "17": 0.369609375, + "18": 0.3044921875, + "19": 0.20428125, + "20": 0.19625 + }, + "per_op_avg_response_len": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375, + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375, + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 621973, + "answer_accuracy": 0.7681258397881758, + "avg_response_len": 265.83570408828643, + "resp_tokens_sum": 215254613, + "pass_at_k": { + "pass@1": 0.7681258397881758, + "pass@2": 0.8056889833794223, + "pass@4": 0.8361066637487128, + "pass@8": 0.8603037310757391, + "pass@16": 0.8792181006261285, + "pass@32": 0.8943473727278064, + "pass@64": 0.9074203405159689, + "pass@128": 0.9190641795763516 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9574575958702065, + "pass@2": 0.9697600196269713, + "pass@4": 0.9770701256369501, + "pass@8": 0.9807352442070735, + "pass@16": 0.9835831385400656, + "pass@32": 0.9868139186902647, + "pass@64": 0.9902929813385202, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996923828125, + "pass@2": 0.9976178026574803, + "pass@4": 0.9982453347628422, + "pass@8": 0.9990349229744625, + "pass@16": 0.9997269200711651, + "pass@32": 0.9999841633446103, + "pass@64": 0.9999999929906153, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9981666354205724, + "pass@4": 0.9995852298224628, + "pass@8": 0.9999710725449563, + "pass@16": 0.9999998592866416, + "pass@32": 0.9999999999993872, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968890344311377, + "pass@2": 0.997264575180348, + "pass@4": 0.997534851556729, + "pass@8": 0.9979835642250304, + "pass@16": 0.9986793120796699, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973187869822485, + "pass@2": 0.9984067785258351, + "pass@4": 0.9992237471795316, + "pass@8": 0.9998073811974663, + "pass@16": 0.9999900064095892, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983974358974359, + "pass@2": 0.999743290051484, + "pass@4": 0.9999842372467864, + "pass@8": 0.9999999472370541, + "pass@16": 0.9999999999998607, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972687007874016, + "pass@4": 0.997876473539399, + "pass@8": 0.9984546377928358, + "pass@16": 0.999188066655105, + "pass@32": 0.9998084093975909, + "pass@64": 0.9999959303192606, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9572193696883853, + "pass@2": 0.9719146628449066, + "pass@4": 0.981724504514839, + "pass@8": 0.9890359200866053, + "pass@16": 0.9932697779011666, + "pass@32": 0.9957361168730702, + "pass@64": 0.9978415915007174, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.8983271539548022, + "pass@2": 0.920239098158281, + "pass@4": 0.9334048053315367, + "pass@8": 0.9434168617466765, + "pass@16": 0.9517176538857137, + "pass@32": 0.9580437536392892, + "pass@64": 0.961533417356335, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8390722962382445, + "pass@2": 0.8873212765038385, + "pass@4": 0.9195430402704361, + "pass@8": 0.9430086769866958, + "pass@16": 0.9574399410309512, + "pass@32": 0.9647881464197946, + "pass@64": 0.9690691600729979, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7840975467289719, + "pass@2": 0.8466142192214293, + "pass@4": 0.8872996447873914, + "pass@8": 0.9131255212050312, + "pass@16": 0.9296828208789675, + "pass@32": 0.9427319318749825, + "pass@64": 0.9554954405969336, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.693099710982659, + "pass@2": 0.7778968583587461, + "pass@4": 0.833728548292012, + "pass@8": 0.8711126510602295, + "pass@16": 0.8958576584940476, + "pass@32": 0.9133306283181201, + "pass@64": 0.9284685661529859, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.4913054435483871, + "pass@2": 0.5962447612395223, + "pass@4": 0.6747601975156323, + "pass@8": 0.7348407821560055, + "pass@16": 0.7834148610294204, + "pass@32": 0.8244895329338998, + "pass@64": 0.8601964216978978, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.35248825867052025, + "pass@2": 0.4572849580128351, + "pass@4": 0.5492719623919837, + "pass@8": 0.6229445136286828, + "pass@16": 0.6790291809157839, + "pass@32": 0.7227658365318381, + "pass@64": 0.7602209527875393, + "pass@128": 0.7976878612716763 + }, + "18": { + "pass@1": 0.27520089285714283, + "pass@2": 0.3652481721034869, + "pass@4": 0.4477995007766887, + "pass@8": 0.5185924991538704, + "pass@16": 0.5777052275710203, + "pass@32": 0.6248875096332369, + "pass@64": 0.6638893307765378, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.2045795795795796, + "pass@2": 0.28665747440353734, + "pass@4": 0.368495601338121, + "pass@8": 0.44158292317157055, + "pass@16": 0.5021246498473418, + "pass@32": 0.5528371750040144, + "pass@64": 0.5987933372999518, + "pass@128": 0.6396396396396397 + }, + "20": { + "pass@1": 0.19566441441441443, + "pass@2": 0.2724017078101725, + "pass@4": 0.34911851418347484, + "pass@8": 0.41767593951939913, + "pass@16": 0.47676148728079454, + "pass@32": 0.5280550430504534, + "pass@64": 0.5749912993919669, + "pass@128": 0.6156156156156156 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 635595, + "answer_accuracy": 0.774058602883866, + "avg_response_len": 275.4775172934528, + "resp_tokens_sum": 226200099, + "pass_at_k": { + "pass@1": 0.774058602883866, + "pass@2": 0.8115189700566369, + "pass@4": 0.8419047259145173, + "pass@8": 0.8661518210454625, + "pass@16": 0.8851545001914215, + "pass@32": 0.9003565916378781, + "pass@64": 0.9130658000646252, + "pass@128": 0.9247077162899454 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421731211656442, + "pass@2": 0.9570635174629243, + "pass@4": 0.9675180118397778, + "pass@8": 0.9744761195433342, + "pass@16": 0.9791587744555608, + "pass@32": 0.9822004591222739, + "pass@64": 0.984571936009979, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956295289855073, + "pass@2": 0.9968182985279016, + "pass@4": 0.9973139007080636, + "pass@8": 0.9976146824957167, + "pass@16": 0.998064779875966, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9972242120343839, + "pass@2": 0.9987002374613632, + "pass@4": 0.9996714137452016, + "pass@8": 0.9999752561101265, + "pass@16": 0.9999998735735035, + "pass@32": 0.999999999999447, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973807803468208, + "pass@2": 0.9977498748350099, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954665492957746, + "pass@2": 0.9968725740268383, + "pass@4": 0.9974011850631346, + "pass@8": 0.9976822657434412, + "pass@16": 0.9981192931743675, + "pass@32": 0.9988210100498002, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930295658682635, + "pass@2": 0.9939206233202886, + "pass@4": 0.9940102609741148, + "pass@8": 0.9940119756217468, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9631024096385542, + "pass@2": 0.972274185323973, + "pass@4": 0.9774970881840069, + "pass@8": 0.9805179334532464, + "pass@16": 0.9822529509986989, + "pass@32": 0.9834096807350399, + "pass@64": 0.9849397431912068, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8928781347962382, + "pass@2": 0.9105734579024021, + "pass@4": 0.920715678859108, + "pass@8": 0.928907449166582, + "pass@16": 0.9358474938634546, + "pass@32": 0.9417129174503823, + "pass@64": 0.9473011819327243, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8309042366946778, + "pass@2": 0.8788773875692008, + "pass@4": 0.9060005905249233, + "pass@8": 0.9230680510501225, + "pass@16": 0.9352975407775181, + "pass@32": 0.9462738870529341, + "pass@64": 0.956931111957274, + "pass@128": 0.9663865546218487 + }, + "14": { + "pass@1": 0.7888797514619883, + "pass@2": 0.8581284966846247, + "pass@4": 0.9010667306937505, + "pass@8": 0.9274503748185816, + "pass@16": 0.9419725164541798, + "pass@32": 0.9505580533617753, + "pass@64": 0.9588922804756361, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.6662642045454545, + "pass@2": 0.7540548198520639, + "pass@4": 0.8157365976980145, + "pass@8": 0.8565477326282641, + "pass@16": 0.8831288202183204, + "pass@32": 0.9034765538582886, + "pass@64": 0.9217986009661605, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.5013799857549858, + "pass@2": 0.6062000162639922, + "pass@4": 0.688734711846916, + "pass@8": 0.7507240841046224, + "pass@16": 0.7988058883708389, + "pass@32": 0.8382289811645698, + "pass@64": 0.8684671985838386, + "pass@128": 0.8945868945868946 + }, + "17": { + "pass@1": 0.39540469720496896, + "pass@2": 0.5015440070670513, + "pass@4": 0.5931639159429605, + "pass@8": 0.6665612968773015, + "pass@16": 0.7219907477028704, + "pass@32": 0.7646644496183821, + "pass@64": 0.7996400240328937, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.32417279411764705, + "pass@2": 0.42494535954145407, + "pass@4": 0.517787185425351, + "pass@8": 0.5978804895419353, + "pass@16": 0.6618267341101194, + "pass@32": 0.7122069429681127, + "pass@64": 0.7544866903270767, + "pass@128": 0.788235294117647 + }, + "19": { + "pass@1": 0.20305524553571427, + "pass@2": 0.2782470794666291, + "pass@4": 0.35577190797578867, + "pass@8": 0.42750031369822505, + "pass@16": 0.4901520073095929, + "pass@32": 0.5430748566756388, + "pass@64": 0.5864859314590565, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.20106661676646706, + "pass@2": 0.2747646930548353, + "pass@4": 0.3491225167774687, + "pass@8": 0.42036866918817706, + "pass@16": 0.4844573511671379, + "pass@32": 0.5371766445282311, + "pass@64": 0.5800651608494923, + "pass@128": 0.6197604790419161 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 614003, + "answer_accuracy": 0.7664001338073174, + "avg_response_len": 296.0467888740214, + "resp_tokens_sum": 237178477, + "pass_at_k": { + "pass@1": 0.7664001338073174, + "pass@2": 0.8041933002303396, + "pass@4": 0.8355785000832348, + "pass@8": 0.8609809020316768, + "pass@16": 0.8807921691495971, + "pass@32": 0.8960189279742334, + "pass@64": 0.9082624863913064, + "pass@128": 0.918517335037546 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9572761194029851, + "pass@2": 0.9666937213538606, + "pass@4": 0.9742225369216908, + "pass@8": 0.9805621470686411, + "pass@16": 0.9864603166226772, + "pass@32": 0.991472929033808, + "pass@64": 0.9949171827192708, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998089774140073, + "pass@4": 0.9999887842966998, + "pass@8": 0.9999999716046218, + "pass@16": 0.9999999999999635, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906479779411764, + "pass@2": 0.9942146248263086, + "pass@4": 0.996329596759964, + "pass@8": 0.9977398649093597, + "pass@16": 0.9989382018700852, + "pass@32": 0.9997259417849353, + "pass@64": 0.9999908945046169, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9988416798107256, + "pass@2": 0.9997768356143968, + "pass@4": 0.9999891550425282, + "pass@8": 0.9999999767890079, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997774920886076, + "pass@2": 0.9992509094986542, + "pass@4": 0.9998849730413445, + "pass@8": 0.9999968489460502, + "pass@16": 0.9999999985879974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9917417417417418, + "pass@2": 0.9956344139415007, + "pass@4": 0.998493621292834, + "pass@8": 0.999787773125201, + "pass@16": 0.9999957560892757, + "pass@32": 0.9999999991534195, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932174437299035, + "pass@2": 0.9964665037850975, + "pass@4": 0.9980255754807175, + "pass@8": 0.9991216467059164, + "pass@16": 0.9997943881339146, + "pass@32": 0.9999917764544182, + "pass@64": 0.9999999987229452, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9642857142857143, + "pass@2": 0.9765787089113863, + "pass@4": 0.9820486778438408, + "pass@8": 0.9842835869687955, + "pass@16": 0.9852484301067794, + "pass@32": 0.9863141673301701, + "pass@64": 0.9881014873140843, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9083763379204893, + "pass@2": 0.9310500644128201, + "pass@4": 0.9455652481513205, + "pass@8": 0.956169735463612, + "pass@16": 0.9645099947486562, + "pass@32": 0.9711630837875163, + "pass@64": 0.9769114489739431, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8012393904320988, + "pass@2": 0.8506990011665209, + "pass@4": 0.8808248887870499, + "pass@8": 0.9035617488598224, + "pass@16": 0.9217366232932731, + "pass@32": 0.9348758028470283, + "pass@64": 0.9448526103401694, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7654395400593472, + "pass@2": 0.8289044428608145, + "pass@4": 0.8680753477773732, + "pass@8": 0.8935025847812458, + "pass@16": 0.9102918510356471, + "pass@32": 0.9225928362575211, + "pass@64": 0.934536393956455, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6340663580246914, + "pass@2": 0.7118602362204722, + "pass@4": 0.7674899159017153, + "pass@8": 0.8127011283278409, + "pass@16": 0.849350961539353, + "pass@32": 0.8759008146277355, + "pass@64": 0.892354569021042, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.4710084808259587, + "pass@2": 0.580898471070541, + "pass@4": 0.668739560762868, + "pass@8": 0.7335904176217118, + "pass@16": 0.7809455072412861, + "pass@32": 0.8166722390431593, + "pass@64": 0.8488481347262333, + "pass@128": 0.8849557522123894 + }, + "17": { + "pass@1": 0.36243411144578314, + "pass@2": 0.4642222927141631, + "pass@4": 0.5592205760612748, + "pass@8": 0.6385525287524542, + "pass@16": 0.6994129926552873, + "pass@32": 0.7462385573378134, + "pass@64": 0.7809706376082151, + "pass@128": 0.8042168674698795 + }, + "18": { + "pass@1": 0.31597782258064516, + "pass@2": 0.4188174688849377, + "pass@4": 0.5104581848639886, + "pass@8": 0.5790573511639918, + "pass@16": 0.6283327210915123, + "pass@32": 0.6653820450438427, + "pass@64": 0.6937567396642746, + "pass@128": 0.7161290322580646 + }, + "19": { + "pass@1": 0.2052256419939577, + "pass@2": 0.28584974248876, + "pass@4": 0.36753850377841757, + "pass@8": 0.44054975863490586, + "pass@16": 0.49795327570192016, + "pass@32": 0.5453148986286661, + "pass@64": 0.5923666332779112, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.19200450450450451, + "pass@2": 0.26821087820103573, + "pass@4": 0.3499301157175173, + "pass@8": 0.4312063902239929, + "pass@16": 0.5043251579531292, + "pass@32": 0.5612592194970087, + "pass@64": 0.6019373120540266, + "pass@128": 0.6306306306306306 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl b/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..62003bd6ecc09fb2d9a0c97b11f01bd52fdcf837 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1290247a50d2ec6142ba64ff7c2f58e783799cc1cf88c8770d0d1e4bdf9569bd +size 2641428571 diff --git a/results/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv b/results/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..568999b6c4c496e0fe924b601b7620b1af7538d9 --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op9-12,0.9921467013888889,0.5692328125,0.7695604440789474,0.0,0.0,0.0,199.46203125,350.66635078125,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block16-epoch4-op9-12/summary.json b/results/composition/SDAR-100M-block16-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3e8fd65e30ad6b8543b7399dff2dff84174c2ace --- /dev/null +++ b/results/composition/SDAR-100M-block16-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op9-12", + "id_acc": 0.9921467013888889, + "ood_acc": 0.5692328125, + "total_acc": 0.7695604440789474, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.46203125, + "ood_avg_resp_len": 350.66635078125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 621973, + "answer_accuracy": 0.7681258397881758, + "avg_response_len": 265.83570408828643, + "resp_tokens_sum": 215254613, + "pass_at_k": { + "pass@1": 0.7681258397881758, + "pass@2": 0.8056889833794223, + "pass@4": 0.8361066637487128, + "pass@8": 0.8603037310757391, + "pass@16": 0.8792181006261285, + "pass@32": 0.8943473727278064, + "pass@64": 0.9074203405159689, + "pass@128": 0.9190641795763516 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9574575958702065, + "pass@2": 0.9697600196269713, + "pass@4": 0.9770701256369501, + "pass@8": 0.9807352442070735, + "pass@16": 0.9835831385400656, + "pass@32": 0.9868139186902647, + "pass@64": 0.9902929813385202, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996923828125, + "pass@2": 0.9976178026574803, + "pass@4": 0.9982453347628422, + "pass@8": 0.9990349229744625, + "pass@16": 0.9997269200711651, + "pass@32": 0.9999841633446103, + "pass@64": 0.9999999929906153, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9981666354205724, + "pass@4": 0.9995852298224628, + "pass@8": 0.9999710725449563, + "pass@16": 0.9999998592866416, + "pass@32": 0.9999999999993872, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968890344311377, + "pass@2": 0.997264575180348, + "pass@4": 0.997534851556729, + "pass@8": 0.9979835642250304, + "pass@16": 0.9986793120796699, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973187869822485, + "pass@2": 0.9984067785258351, + "pass@4": 0.9992237471795316, + "pass@8": 0.9998073811974663, + "pass@16": 0.9999900064095892, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983974358974359, + "pass@2": 0.999743290051484, + "pass@4": 0.9999842372467864, + "pass@8": 0.9999999472370541, + "pass@16": 0.9999999999998607, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972687007874016, + "pass@4": 0.997876473539399, + "pass@8": 0.9984546377928358, + "pass@16": 0.999188066655105, + "pass@32": 0.9998084093975909, + "pass@64": 0.9999959303192606, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9572193696883853, + "pass@2": 0.9719146628449066, + "pass@4": 0.981724504514839, + "pass@8": 0.9890359200866053, + "pass@16": 0.9932697779011666, + "pass@32": 0.9957361168730702, + "pass@64": 0.9978415915007174, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.8983271539548022, + "pass@2": 0.920239098158281, + "pass@4": 0.9334048053315367, + "pass@8": 0.9434168617466765, + "pass@16": 0.9517176538857137, + "pass@32": 0.9580437536392892, + "pass@64": 0.961533417356335, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8390722962382445, + "pass@2": 0.8873212765038385, + "pass@4": 0.9195430402704361, + "pass@8": 0.9430086769866958, + "pass@16": 0.9574399410309512, + "pass@32": 0.9647881464197946, + "pass@64": 0.9690691600729979, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7840975467289719, + "pass@2": 0.8466142192214293, + "pass@4": 0.8872996447873914, + "pass@8": 0.9131255212050312, + "pass@16": 0.9296828208789675, + "pass@32": 0.9427319318749825, + "pass@64": 0.9554954405969336, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.693099710982659, + "pass@2": 0.7778968583587461, + "pass@4": 0.833728548292012, + "pass@8": 0.8711126510602295, + "pass@16": 0.8958576584940476, + "pass@32": 0.9133306283181201, + "pass@64": 0.9284685661529859, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.4913054435483871, + "pass@2": 0.5962447612395223, + "pass@4": 0.6747601975156323, + "pass@8": 0.7348407821560055, + "pass@16": 0.7834148610294204, + "pass@32": 0.8244895329338998, + "pass@64": 0.8601964216978978, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.35248825867052025, + "pass@2": 0.4572849580128351, + "pass@4": 0.5492719623919837, + "pass@8": 0.6229445136286828, + "pass@16": 0.6790291809157839, + "pass@32": 0.7227658365318381, + "pass@64": 0.7602209527875393, + "pass@128": 0.7976878612716763 + }, + "18": { + "pass@1": 0.27520089285714283, + "pass@2": 0.3652481721034869, + "pass@4": 0.4477995007766887, + "pass@8": 0.5185924991538704, + "pass@16": 0.5777052275710203, + "pass@32": 0.6248875096332369, + "pass@64": 0.6638893307765378, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.2045795795795796, + "pass@2": 0.28665747440353734, + "pass@4": 0.368495601338121, + "pass@8": 0.44158292317157055, + "pass@16": 0.5021246498473418, + "pass@32": 0.5528371750040144, + "pass@64": 0.5987933372999518, + "pass@128": 0.6396396396396397 + }, + "20": { + "pass@1": 0.19566441441441443, + "pass@2": 0.2724017078101725, + "pass@4": 0.34911851418347484, + "pass@8": 0.41767593951939913, + "pass@16": 0.47676148728079454, + "pass@32": 0.5280550430504534, + "pass@64": 0.5749912993919669, + "pass@128": 0.6156156156156156 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 635595, + "answer_accuracy": 0.774058602883866, + "avg_response_len": 275.4775172934528, + "resp_tokens_sum": 226200099, + "pass_at_k": { + "pass@1": 0.774058602883866, + "pass@2": 0.8115189700566369, + "pass@4": 0.8419047259145173, + "pass@8": 0.8661518210454625, + "pass@16": 0.8851545001914215, + "pass@32": 0.9003565916378781, + "pass@64": 0.9130658000646252, + "pass@128": 0.9247077162899454 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421731211656442, + "pass@2": 0.9570635174629243, + "pass@4": 0.9675180118397778, + "pass@8": 0.9744761195433342, + "pass@16": 0.9791587744555608, + "pass@32": 0.9822004591222739, + "pass@64": 0.984571936009979, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956295289855073, + "pass@2": 0.9968182985279016, + "pass@4": 0.9973139007080636, + "pass@8": 0.9976146824957167, + "pass@16": 0.998064779875966, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9972242120343839, + "pass@2": 0.9987002374613632, + "pass@4": 0.9996714137452016, + "pass@8": 0.9999752561101265, + "pass@16": 0.9999998735735035, + "pass@32": 0.999999999999447, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973807803468208, + "pass@2": 0.9977498748350099, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954665492957746, + "pass@2": 0.9968725740268383, + "pass@4": 0.9974011850631346, + "pass@8": 0.9976822657434412, + "pass@16": 0.9981192931743675, + "pass@32": 0.9988210100498002, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930295658682635, + "pass@2": 0.9939206233202886, + "pass@4": 0.9940102609741148, + "pass@8": 0.9940119756217468, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9631024096385542, + "pass@2": 0.972274185323973, + "pass@4": 0.9774970881840069, + "pass@8": 0.9805179334532464, + "pass@16": 0.9822529509986989, + "pass@32": 0.9834096807350399, + "pass@64": 0.9849397431912068, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8928781347962382, + "pass@2": 0.9105734579024021, + "pass@4": 0.920715678859108, + "pass@8": 0.928907449166582, + "pass@16": 0.9358474938634546, + "pass@32": 0.9417129174503823, + "pass@64": 0.9473011819327243, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8309042366946778, + "pass@2": 0.8788773875692008, + "pass@4": 0.9060005905249233, + "pass@8": 0.9230680510501225, + "pass@16": 0.9352975407775181, + "pass@32": 0.9462738870529341, + "pass@64": 0.956931111957274, + "pass@128": 0.9663865546218487 + }, + "14": { + "pass@1": 0.7888797514619883, + "pass@2": 0.8581284966846247, + "pass@4": 0.9010667306937505, + "pass@8": 0.9274503748185816, + "pass@16": 0.9419725164541798, + "pass@32": 0.9505580533617753, + "pass@64": 0.9588922804756361, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.6662642045454545, + "pass@2": 0.7540548198520639, + "pass@4": 0.8157365976980145, + "pass@8": 0.8565477326282641, + "pass@16": 0.8831288202183204, + "pass@32": 0.9034765538582886, + "pass@64": 0.9217986009661605, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.5013799857549858, + "pass@2": 0.6062000162639922, + "pass@4": 0.688734711846916, + "pass@8": 0.7507240841046224, + "pass@16": 0.7988058883708389, + "pass@32": 0.8382289811645698, + "pass@64": 0.8684671985838386, + "pass@128": 0.8945868945868946 + }, + "17": { + "pass@1": 0.39540469720496896, + "pass@2": 0.5015440070670513, + "pass@4": 0.5931639159429605, + "pass@8": 0.6665612968773015, + "pass@16": 0.7219907477028704, + "pass@32": 0.7646644496183821, + "pass@64": 0.7996400240328937, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.32417279411764705, + "pass@2": 0.42494535954145407, + "pass@4": 0.517787185425351, + "pass@8": 0.5978804895419353, + "pass@16": 0.6618267341101194, + "pass@32": 0.7122069429681127, + "pass@64": 0.7544866903270767, + "pass@128": 0.788235294117647 + }, + "19": { + "pass@1": 0.20305524553571427, + "pass@2": 0.2782470794666291, + "pass@4": 0.35577190797578867, + "pass@8": 0.42750031369822505, + "pass@16": 0.4901520073095929, + "pass@32": 0.5430748566756388, + "pass@64": 0.5864859314590565, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.20106661676646706, + "pass@2": 0.2747646930548353, + "pass@4": 0.3491225167774687, + "pass@8": 0.42036866918817706, + "pass@16": 0.4844573511671379, + "pass@32": 0.5371766445282311, + "pass@64": 0.5800651608494923, + "pass@128": 0.6197604790419161 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 614003, + "answer_accuracy": 0.7664001338073174, + "avg_response_len": 296.0467888740214, + "resp_tokens_sum": 237178477, + "pass_at_k": { + "pass@1": 0.7664001338073174, + "pass@2": 0.8041933002303396, + "pass@4": 0.8355785000832348, + "pass@8": 0.8609809020316768, + "pass@16": 0.8807921691495971, + "pass@32": 0.8960189279742334, + "pass@64": 0.9082624863913064, + "pass@128": 0.918517335037546 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9572761194029851, + "pass@2": 0.9666937213538606, + "pass@4": 0.9742225369216908, + "pass@8": 0.9805621470686411, + "pass@16": 0.9864603166226772, + "pass@32": 0.991472929033808, + "pass@64": 0.9949171827192708, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998089774140073, + "pass@4": 0.9999887842966998, + "pass@8": 0.9999999716046218, + "pass@16": 0.9999999999999635, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906479779411764, + "pass@2": 0.9942146248263086, + "pass@4": 0.996329596759964, + "pass@8": 0.9977398649093597, + "pass@16": 0.9989382018700852, + "pass@32": 0.9997259417849353, + "pass@64": 0.9999908945046169, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9988416798107256, + "pass@2": 0.9997768356143968, + "pass@4": 0.9999891550425282, + "pass@8": 0.9999999767890079, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997774920886076, + "pass@2": 0.9992509094986542, + "pass@4": 0.9998849730413445, + "pass@8": 0.9999968489460502, + "pass@16": 0.9999999985879974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9917417417417418, + "pass@2": 0.9956344139415007, + "pass@4": 0.998493621292834, + "pass@8": 0.999787773125201, + "pass@16": 0.9999957560892757, + "pass@32": 0.9999999991534195, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932174437299035, + "pass@2": 0.9964665037850975, + "pass@4": 0.9980255754807175, + "pass@8": 0.9991216467059164, + "pass@16": 0.9997943881339146, + "pass@32": 0.9999917764544182, + "pass@64": 0.9999999987229452, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9642857142857143, + "pass@2": 0.9765787089113863, + "pass@4": 0.9820486778438408, + "pass@8": 0.9842835869687955, + "pass@16": 0.9852484301067794, + "pass@32": 0.9863141673301701, + "pass@64": 0.9881014873140843, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9083763379204893, + "pass@2": 0.9310500644128201, + "pass@4": 0.9455652481513205, + "pass@8": 0.956169735463612, + "pass@16": 0.9645099947486562, + "pass@32": 0.9711630837875163, + "pass@64": 0.9769114489739431, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8012393904320988, + "pass@2": 0.8506990011665209, + "pass@4": 0.8808248887870499, + "pass@8": 0.9035617488598224, + "pass@16": 0.9217366232932731, + "pass@32": 0.9348758028470283, + "pass@64": 0.9448526103401694, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7654395400593472, + "pass@2": 0.8289044428608145, + "pass@4": 0.8680753477773732, + "pass@8": 0.8935025847812458, + "pass@16": 0.9102918510356471, + "pass@32": 0.9225928362575211, + "pass@64": 0.934536393956455, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6340663580246914, + "pass@2": 0.7118602362204722, + "pass@4": 0.7674899159017153, + "pass@8": 0.8127011283278409, + "pass@16": 0.849350961539353, + "pass@32": 0.8759008146277355, + "pass@64": 0.892354569021042, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.4710084808259587, + "pass@2": 0.580898471070541, + "pass@4": 0.668739560762868, + "pass@8": 0.7335904176217118, + "pass@16": 0.7809455072412861, + "pass@32": 0.8166722390431593, + "pass@64": 0.8488481347262333, + "pass@128": 0.8849557522123894 + }, + "17": { + "pass@1": 0.36243411144578314, + "pass@2": 0.4642222927141631, + "pass@4": 0.5592205760612748, + "pass@8": 0.6385525287524542, + "pass@16": 0.6994129926552873, + "pass@32": 0.7462385573378134, + "pass@64": 0.7809706376082151, + "pass@128": 0.8042168674698795 + }, + "18": { + "pass@1": 0.31597782258064516, + "pass@2": 0.4188174688849377, + "pass@4": 0.5104581848639886, + "pass@8": 0.5790573511639918, + "pass@16": 0.6283327210915123, + "pass@32": 0.6653820450438427, + "pass@64": 0.6937567396642746, + "pass@128": 0.7161290322580646 + }, + "19": { + "pass@1": 0.2052256419939577, + "pass@2": 0.28584974248876, + "pass@4": 0.36753850377841757, + "pass@8": 0.44054975863490586, + "pass@16": 0.49795327570192016, + "pass@32": 0.5453148986286661, + "pass@64": 0.5923666332779112, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.19200450450450451, + "pass@2": 0.26821087820103573, + "pass@4": 0.3499301157175173, + "pass@8": 0.4312063902239929, + "pass@16": 0.5043251579531292, + "pass@32": 0.5612592194970087, + "pass@64": 0.6019373120540266, + "pass@128": 0.6306306306306306 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e51515db616b34ad341d6835c28c8dd56034b417 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db9f6d58176cd3599c8421eff106c2759c02b9f77c7871f8743386df77e636e +size 1791373215 diff --git a/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json b/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dee706e673251e2ce0e53bbb20faa106bff1a6e7 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op11-14", + "id": { + "answer_accuracy": 0.9827213541666666, + "avg_response_len": 199.76456336805555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9827213541666666, + "pass@2": 0.9873874945319294, + "pass@4": 0.9899934680664909, + "pass@8": 0.9919332484253088, + "pass@16": 0.993608891907506, + "pass@32": 0.9950211980869957, + "pass@64": 0.9962416535993714, + "pass@128": 0.9974444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.92821875, + "pass@2": 0.9428015501968511, + "pass@4": 0.95275325712411, + "pass@8": 0.9602836900431464, + "pass@16": 0.9667643456431163, + "pass@32": 0.9724262544552759, + "pass@64": 0.9773916206622802, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99825, + "pass@2": 0.9988725393700788, + "pass@4": 0.9990494026996625, + "pass@8": 0.9991214535331311, + "pass@16": 0.9992352362176316, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99721875, + "pass@2": 0.9981986958661418, + "pass@4": 0.99854321287964, + "pass@8": 0.9989217378407179, + "pass@16": 0.9993741635872916, + "pass@32": 0.9997590674152546, + "pass@64": 0.9999711690328693, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980671875, + "pass@2": 0.9855286663385826, + "pass@4": 0.9891177830896137, + "pass@8": 0.9923601897966485, + "pass@16": 0.9951406369282718, + "pass@32": 0.9969241199883416, + "pass@64": 0.9980015759808923, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9930234375, + "pass@2": 0.994091658464567, + "pass@4": 0.9947228139295088, + "pass@8": 0.9952883871401448, + "pass@16": 0.9959038226355749, + "pass@32": 0.9965991523224128, + "pass@64": 0.9973491337914664, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9889375, + "pass@2": 0.9931058070866141, + "pass@4": 0.9949000196850394, + "pass@8": 0.9961006060233324, + "pass@16": 0.9972014482074887, + "pass@32": 0.998169276945154, + "pass@64": 0.99912682009508, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9813515625, + "pass@2": 0.9889160925196855, + "pass@4": 0.9925639879077615, + "pass@8": 0.9947117324737054, + "pass@16": 0.9964938152859911, + "pass@32": 0.9980639650979307, + "pass@64": 0.9992073394908063, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9768671875, + "pass@2": 0.9849725639763786, + "pass@4": 0.9882907352830895, + "pass@8": 0.9906114389769584, + "pass@16": 0.9923665586621832, + "pass@32": 0.9938099701806373, + "pass@64": 0.9953752548370095, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.92821875, + "2": 0.999953125, + "3": 0.99825, + "4": 0.99721875, + "5": 0.980671875, + "6": 0.9930234375, + "7": 0.9889375, + "8": 0.9813515625, + "9": 0.9768671875 + }, + "per_op_avg_response_len": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 372242, + "answer_accuracy": 0.9821481340763256, + "avg_response_len": 190.2342826536643, + "resp_tokens_sum": 72100315, + "pass_at_k": { + "pass@1": 0.9821481340763256, + "pass@2": 0.9872708730557621, + "pass@4": 0.9901135191497727, + "pass@8": 0.9920274304013039, + "pass@16": 0.9934861887733804, + "pass@32": 0.9946622220879497, + "pass@64": 0.9956885677389574, + "pass@128": 0.9966227625802094 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9259080014749262, + "pass@2": 0.9433224020393468, + "pass@4": 0.9555353711317055, + "pass@8": 0.9626092113862927, + "pass@16": 0.9672899140408969, + "pass@32": 0.971873515401767, + "pass@64": 0.9768429344531748, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999044342507645, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9946638933316929, + "pass@4": 0.995454402184102, + "pass@8": 0.9966304348689696, + "pass@16": 0.9980442612102862, + "pass@32": 0.9992470856726705, + "pass@64": 0.9999099032277166, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9814980158730159, + "pass@2": 0.9859626921634798, + "pass@4": 0.9891130867570126, + "pass@8": 0.9915862161820935, + "pass@16": 0.9933890901170934, + "pass@32": 0.9943969337435874, + "pass@64": 0.9952380498015997, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.992374625748503, + "pass@2": 0.9937386545806025, + "pass@4": 0.9939952238380383, + "pass@8": 0.9940119126755125, + "pass@16": 0.9940119760476699, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9904539571005917, + "pass@2": 0.9942641114941992, + "pass@4": 0.995715497378656, + "pass@8": 0.9967252188259338, + "pass@16": 0.9975999557269617, + "pass@32": 0.9983364053352927, + "pass@64": 0.9992661788080938, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9777393830128205, + "pass@2": 0.9862255987532803, + "pass@4": 0.990327881490775, + "pass@8": 0.9937928780958446, + "pass@16": 0.996912117506564, + "pass@32": 0.9990931142481682, + "pass@64": 0.9999455717474616, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9797535211267606, + "pass@2": 0.9886772901186649, + "pass@4": 0.9919861029695232, + "pass@8": 0.9939178049019348, + "pass@16": 0.9951668490880456, + "pass@32": 0.996002421656276, + "pass@64": 0.9968393035287623, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383958, + "answer_accuracy": 0.9828544806684142, + "avg_response_len": 197.82942793659896, + "resp_tokens_sum": 77283253, + "pass_at_k": { + "pass@1": 0.9828544806684142, + "pass@2": 0.9864802011202161, + "pass@4": 0.9885723154608951, + "pass@8": 0.9902874693103434, + "pass@16": 0.9919324015926441, + "pass@32": 0.9934557354166121, + "pass@64": 0.9949606718992721, + "pass@128": 0.9967234600262124 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163391487730062, + "pass@2": 0.9258629564996861, + "pass@4": 0.9336692571212339, + "pass@8": 0.9416640558086454, + "pass@16": 0.9500473404479759, + "pass@32": 0.9578994674031669, + "pass@64": 0.9655855093279391, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9995746092817943, + "pass@4": 0.9999632247673588, + "pass@8": 0.9999996922182559, + "pass@16": 0.9999999999913911, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986131656804734, + "pass@2": 0.999854036597866, + "pass@4": 0.9999976565429319, + "pass@8": 0.9999999993354288, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863224637681159, + "pass@2": 0.9906003937007875, + "pass@4": 0.9930274229308294, + "pass@8": 0.9950581299215392, + "pass@16": 0.9969160592890997, + "pass@32": 0.998278697645461, + "pass@64": 0.9992808085046374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967765042979942, + "pass@2": 0.9973253869322924, + "pass@4": 0.9975613690301607, + "pass@8": 0.9979359856831456, + "pass@16": 0.9985470770655008, + "pass@32": 0.9993380089666443, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9916230130057804, + "pass@2": 0.9936738342587959, + "pass@4": 0.9944413127911034, + "pass@8": 0.9949076292657711, + "pass@16": 0.9955414667011108, + "pass@32": 0.9966227126233496, + "pass@64": 0.9982021755951027, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9815140845070423, + "pass@2": 0.9889029056227125, + "pass@4": 0.9929117410499744, + "pass@8": 0.9949431586627508, + "pass@16": 0.9962341274090126, + "pass@32": 0.9974052544005088, + "pass@64": 0.9985526092972625, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9739427395209581, + "pass@2": 0.9802412890753925, + "pass@4": 0.9834003469439074, + "pass@8": 0.9861000031633753, + "pass@16": 0.9883617473597814, + "pass@32": 0.9900186779076805, + "pass@64": 0.9917721721639942, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375895, + "answer_accuracy": 0.9831535612654838, + "avg_response_len": 211.18913468781386, + "resp_tokens_sum": 80745209, + "pass_at_k": { + "pass@1": 0.9831535612654838, + "pass@2": 0.9884301378809491, + "pass@4": 0.9913265402106796, + "pass@8": 0.9935214791008893, + "pass@16": 0.995443499346777, + "pass@32": 0.9969765780679748, + "pass@64": 0.9980987822841344, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421175373134328, + "pass@2": 0.9587580062286991, + "pass@4": 0.9685092193699665, + "pass@8": 0.9760498065360441, + "pass@16": 0.9825003934245147, + "pass@32": 0.9871221085392369, + "pass@64": 0.9894357905723764, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999330357142857, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9971243629622828, + "pass@4": 0.9972639609917566, + "pass@8": 0.99743893615483, + "pass@16": 0.9977703679897155, + "pass@32": 0.9983643626179382, + "pass@64": 0.9992768761047727, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989720394736842, + "pass@2": 0.9998701334208223, + "pass@4": 0.9999958955240242, + "pass@8": 0.9999999968048913, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.974172794117647, + "pass@2": 0.9799802425891616, + "pass@4": 0.9851549995588786, + "pass@8": 0.9903396378716414, + "pass@16": 0.9949618620195534, + "pass@32": 0.9978909857983171, + "pass@64": 0.9992638569390833, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.98957511829653, + "pass@2": 0.9909034315308379, + "pass@4": 0.9923643261075016, + "pass@8": 0.9937184520602704, + "pass@16": 0.9949870433431968, + "pass@32": 0.9963097387793499, + "pass@64": 0.9980377056438621, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.984375, + "pass@2": 0.9912449074304795, + "pass@4": 0.994530023244721, + "pass@8": 0.9967387415639558, + "pass@16": 0.9985927711809854, + "pass@32": 0.9996839030827411, + "pass@64": 0.9999901863988568, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9845626876876877, + "pass@2": 0.9914509736114068, + "pass@4": 0.9942883507804767, + "pass@8": 0.9953259254730494, + "pass@16": 0.9963787369181493, + "pass@32": 0.9978019463673319, + "pass@64": 0.9992136480632735, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9767132234726688, + "pass@2": 0.9858248727751473, + "pass@4": 0.9893246072335813, + "pass@8": 0.9916823703543539, + "pass@16": 0.9934710727259167, + "pass@32": 0.995379009876187, + "pass@64": 0.9975736224486328, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.615015625, + "avg_response_len": 367.26526484375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.615015625, + "pass@2": 0.688048412893687, + "pass@4": 0.7440958128702582, + "pass@8": 0.7877747365946025, + "pass@16": 0.8222755379094328, + "pass@32": 0.8496575803035179, + "pass@64": 0.8714758455012228, + "pass@128": 0.8895 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9379609375, + "pass@2": 0.9578695866141734, + "pass@4": 0.9690707025684286, + "pass@8": 0.9761228584925753, + "pass@16": 0.9809705725972232, + "pass@32": 0.9846269650875722, + "pass@64": 0.9872749236211029, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.89421875, + "pass@2": 0.9220222687007882, + "pass@4": 0.9362151971316084, + "pass@8": 0.9447333294059417, + "pass@16": 0.9509831248030044, + "pass@32": 0.9562493532460535, + "pass@64": 0.961528048501336, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.8007421875, + "pass@2": 0.850632997047245, + "pass@4": 0.8801703857330323, + "pass@8": 0.900917953033891, + "pass@16": 0.9170591607756168, + "pass@32": 0.9292019660423374, + "pass@64": 0.9391975690385227, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.777703125, + "pass@2": 0.8398607283464579, + "pass@4": 0.8762118897637787, + "pass@8": 0.9005872635175092, + "pass@16": 0.9194266016183247, + "pass@32": 0.9342833957716893, + "pass@64": 0.9452691557195909, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.69453125, + "pass@2": 0.768718626968505, + "pass@4": 0.8212702295650541, + "pass@8": 0.8608277460887901, + "pass@16": 0.8904122978547405, + "pass@32": 0.9119605369492704, + "pass@64": 0.927476651698037, + "pass@128": 0.938 + }, + "16": { + "pass@1": 0.5662421875, + "pass@2": 0.6677634104330711, + "pass@4": 0.7407699299775025, + "pass@8": 0.7935083278076046, + "pass@16": 0.8317948982183991, + "pass@32": 0.8604213110568941, + "pass@64": 0.8841963329743866, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.4808125, + "pass@2": 0.585482406496063, + "pass@4": 0.6681984115110617, + "pass@8": 0.7304857882018998, + "pass@16": 0.7780592856710695, + "pass@32": 0.8142376854501494, + "pass@64": 0.8410238580115026, + "pass@128": 0.864 + }, + "18": { + "pass@1": 0.44084375, + "pass@2": 0.5405799704724411, + "pass@4": 0.6218796379827516, + "pass@8": 0.6872290020729789, + "pass@16": 0.7397756567660226, + "pass@32": 0.7814495225157587, + "pass@64": 0.8142298574628336, + "pass@128": 0.84 + }, + "19": { + "pass@1": 0.2833359375, + "pass@2": 0.37453592519685047, + "pass@4": 0.4594816270153731, + "pass@8": 0.5354858260969473, + "pass@16": 0.6025362039793238, + "pass@32": 0.6610735591981041, + "pass@64": 0.710950493166852, + "pass@128": 0.753 + }, + "20": { + "pass@1": 0.273765625, + "pass@2": 0.37301820866141744, + "pass@4": 0.4676901174540681, + "pass@8": 0.5478492712279519, + "pass@16": 0.6117375768104771, + "pass@32": 0.6630715077173114, + "pass@64": 0.7036115648179885, + "pass@128": 0.736 + } + }, + "per_op_accuracy": { + "11": 0.9379609375, + "12": 0.89421875, + "13": 0.8007421875, + "14": 0.777703125, + "15": 0.69453125, + "16": 0.5662421875, + "17": 0.4808125, + "18": 0.44084375, + "19": 0.2833359375, + "20": 0.273765625 + }, + "per_op_avg_response_len": { + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 268071, + "answer_accuracy": 0.6223788075780089, + "avg_response_len": 348.7750650074294, + "resp_tokens_sum": 150224396, + "pass_at_k": { + "pass@1": 0.6223788075780089, + "pass@2": 0.6935501880754923, + "pass@4": 0.7469774392428321, + "pass@8": 0.7885196429188797, + "pass@16": 0.8221776320825549, + "pass@32": 0.8495236946886684, + "pass@64": 0.8715822180293861, + "pass@128": 0.8900445765230312 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9369688385269122, + "pass@2": 0.9556891436729048, + "pass@4": 0.9681532095138248, + "pass@8": 0.9773977476860249, + "pass@16": 0.9836760616393073, + "pass@32": 0.9879154034727532, + "pass@64": 0.9905512293835983, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.896340925141243, + "pass@2": 0.9266628019484855, + "pass@4": 0.9416773099337153, + "pass@8": 0.9499502893000226, + "pass@16": 0.9556055452848286, + "pass@32": 0.9601108862491841, + "pass@64": 0.9653730933928352, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8189410266457681, + "pass@2": 0.8701832590773324, + "pass@4": 0.9008475028003936, + "pass@8": 0.9223142658352539, + "pass@16": 0.9375126996972292, + "pass@32": 0.9461958750908102, + "pass@64": 0.9519937435789704, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8048091900311527, + "pass@2": 0.8633829138764197, + "pass@4": 0.8936096031220395, + "pass@8": 0.9133137052944442, + "pass@16": 0.9308204065370922, + "pass@32": 0.9458697424997475, + "pass@64": 0.9578949685863664, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7142792630057804, + "pass@2": 0.7852296777570432, + "pass@4": 0.8336160666874729, + "pass@8": 0.8703838255639478, + "pass@16": 0.8991492741726431, + "pass@32": 0.9217285219094363, + "pass@64": 0.9390414573776548, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5767389112903226, + "pass@2": 0.671137207899416, + "pass@4": 0.7352816832371761, + "pass@8": 0.7801929643943304, + "pass@16": 0.8153526018049102, + "pass@32": 0.8451755884363006, + "pass@64": 0.8721477928442585, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.49742593930635837, + "pass@2": 0.6042979950844295, + "pass@4": 0.6877961173386561, + "pass@8": 0.750236429513258, + "pass@16": 0.797521632533749, + "pass@32": 0.8336931159590578, + "pass@64": 0.861816202176084, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.4068080357142857, + "pass@2": 0.5067449381327332, + "pass@4": 0.5904474240719905, + "pass@8": 0.6602663145937516, + "pass@16": 0.7184114317717843, + "pass@32": 0.7636522860956406, + "pass@64": 0.7975986551347283, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.28638795045045046, + "pass@2": 0.37785669527795496, + "pass@4": 0.45953530245656216, + "pass@8": 0.5290022873626415, + "pass@16": 0.5902007338149695, + "pass@32": 0.6466124054432371, + "pass@64": 0.6966004194500958, + "pass@128": 0.7417417417417418 + }, + "20": { + "pass@1": 0.2728978978978979, + "pass@2": 0.36531228571090785, + "pass@4": 0.4511123266123264, + "pass@8": 0.5250073329043563, + "pass@16": 0.5869595546727128, + "pass@32": 0.638416275402605, + "pass@64": 0.6777993410094819, + "pass@128": 0.7057057057057057 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 265030, + "answer_accuracy": 0.615684470710675, + "avg_response_len": 363.1408782151353, + "resp_tokens_sum": 156319075, + "pass_at_k": { + "pass@1": 0.615684470710675, + "pass@2": 0.6895557930091535, + "pass@4": 0.7462455992343082, + "pass@8": 0.7906967253680187, + "pass@16": 0.8259765555967202, + "pass@32": 0.8537529719929998, + "pass@64": 0.875734930364443, + "pass@128": 0.8941421349985132 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.942371046686747, + "pass@2": 0.9614907711554884, + "pass@4": 0.9709818822534232, + "pass@8": 0.9762108984873451, + "pass@16": 0.9798418328465113, + "pass@32": 0.9832802016718227, + "pass@64": 0.9869836840656171, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8921189263322884, + "pass@2": 0.9154603923431991, + "pass@4": 0.9262470747191082, + "pass@8": 0.932674949530915, + "pass@16": 0.9378135612950852, + "pass@32": 0.942572203878569, + "pass@64": 0.947465209542479, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7801995798319328, + "pass@2": 0.8263980789166057, + "pass@4": 0.8551042666410398, + "pass@8": 0.8787522658307619, + "pass@16": 0.8997253159536589, + "pass@32": 0.9175379858563458, + "pass@64": 0.9326565965286922, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7647112573099415, + "pass@2": 0.8316522626283556, + "pass@4": 0.8720206605205054, + "pass@8": 0.899213977549253, + "pass@16": 0.9185417582132726, + "pass@32": 0.9327907985067769, + "pass@64": 0.9426059507605182, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.6902225378787878, + "pass@2": 0.7702934860415173, + "pass@4": 0.8277060339048522, + "pass@8": 0.870181513451221, + "pass@16": 0.9009560991102984, + "pass@32": 0.9213126526056608, + "pass@64": 0.9341966745085428, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5663951210826211, + "pass@2": 0.6729072728088475, + "pass@4": 0.747877350267114, + "pass@8": 0.8014397473398795, + "pass@16": 0.8403655118855234, + "pass@32": 0.8670101699239859, + "pass@64": 0.8852311741120641, + "pass@128": 0.9002849002849003 + }, + "17": { + "pass@1": 0.499708850931677, + "pass@2": 0.6042562020100748, + "pass@4": 0.6857853701905272, + "pass@8": 0.7446535294212936, + "pass@16": 0.787459898111565, + "pass@32": 0.8184923783964188, + "pass@64": 0.840558376080876, + "pass@128": 0.860248447204969 + }, + "18": { + "pass@1": 0.47116268382352944, + "pass@2": 0.5732706837656322, + "pass@4": 0.6578277650036389, + "pass@8": 0.7272127940450731, + "pass@16": 0.7824553129641325, + "pass@32": 0.8258777832780535, + "pass@64": 0.858667717455225, + "pass@128": 0.8823529411764706 + }, + "19": { + "pass@1": 0.2761346726190476, + "pass@2": 0.365666449896888, + "pass@4": 0.45070808615440894, + "pass@8": 0.529863381029973, + "pass@16": 0.6031771625339937, + "pass@32": 0.6680454701301732, + "pass@64": 0.7240389650730391, + "pass@128": 0.7708333333333334 + }, + "20": { + "pass@1": 0.277156624251497, + "pass@2": 0.3758873738742987, + "pass@4": 0.46919662278367874, + "pass@8": 0.547041580122116, + "pass@16": 0.608975135388662, + "pass@32": 0.6595693283084222, + "pass@64": 0.7034790052169011, + "pass@128": 0.7455089820359282 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 254119, + "answer_accuracy": 0.6067557113386308, + "avg_response_len": 390.5201042940098, + "resp_tokens_sum": 163556068, + "pass_at_k": { + "pass@1": 0.6067557113386308, + "pass@2": 0.6808409578769081, + "pass@4": 0.7389227064259096, + "pass@8": 0.7840054034570462, + "pass@16": 0.818572277098011, + "pass@32": 0.8455859797051636, + "pass@64": 0.8669889121416895, + "pass@128": 0.8841687041564792 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9344246031746032, + "pass@2": 0.956496453568304, + "pass@4": 0.9680845546092448, + "pass@8": 0.9746013817828894, + "pass@16": 0.979128362963511, + "pass@32": 0.9823612720845869, + "pass@64": 0.9839103382187553, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8939698012232415, + "pass@2": 0.9233999133135886, + "pass@4": 0.940026338164151, + "pass@8": 0.9508489849950189, + "pass@16": 0.9588264089267363, + "pass@32": 0.9654115182769986, + "pass@64": 0.9710843168080171, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8054591049382716, + "pass@2": 0.8580877260134149, + "pass@4": 0.8874315097186924, + "pass@8": 0.9042751337372295, + "pass@16": 0.9160205919035335, + "pass@32": 0.9253222868446103, + "pass@64": 0.9338060490002721, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7650686201780416, + "pass@2": 0.8257855761816872, + "pass@4": 0.8638935942539817, + "pass@8": 0.8898587056265527, + "pass@16": 0.909471720507359, + "pass@32": 0.9247618852817582, + "pass@64": 0.9359455063598505, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6778308256172839, + "pass@2": 0.7494824323174885, + "pass@4": 0.801531108958602, + "pass@8": 0.8410958117430899, + "pass@16": 0.8703430132240998, + "pass@32": 0.8920039907677088, + "pass@64": 0.9082821137578678, + "pass@128": 0.9197530864197531 + }, + "16": { + "pass@1": 0.5564850663716814, + "pass@2": 0.659352280909577, + "pass@4": 0.7384296702956372, + "pass@8": 0.7974724410886839, + "pass@16": 0.8379566282804083, + "pass@32": 0.867540734508324, + "pass@64": 0.8941426993490582, + "pass@128": 0.9174041297935103 + }, + "17": { + "pass@1": 0.44517131024096385, + "pass@2": 0.5476650697277298, + "pass@4": 0.630717065393181, + "pass@8": 0.696161286495951, + "pass@16": 0.7486587759712936, + "pass@32": 0.7898353056767135, + "pass@64": 0.8198061896401653, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.44601814516129035, + "pass@2": 0.5429264827279653, + "pass@4": 0.6179406434074768, + "pass@8": 0.6738175548059356, + "pass@16": 0.7170866104454663, + "pass@32": 0.7528154066701483, + "pass@64": 0.7842687233254886, + "pass@128": 0.8129032258064516 + }, + "19": { + "pass@1": 0.28757552870090636, + "pass@2": 0.3801985453291147, + "pass@4": 0.4683336989409553, + "pass@8": 0.5477159165532208, + "pass@16": 0.6142955679985431, + "pass@32": 0.6685448647183322, + "pass@64": 0.7121010913154943, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2712321696696697, + "pass@2": 0.377846350287295, + "pass@4": 0.48275687892617025, + "pass@8": 0.5715013261561981, + "pass@16": 0.6392863359899448, + "pass@32": 0.6912394364961897, + "pass@64": 0.7295567463044594, + "pass@128": 0.7567567567567568 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7891920230263157, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7891920230263157, + "pass@2": 0.8298406094591582, + "pass@4": 0.8605736495421483, + "pass@8": 0.8844814000933748, + "pass@16": 0.9034334424348239, + "pass@32": 0.9185140308325449, + "pass@64": 0.930575438810868, + "pass@128": 0.9406315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.92821875, + "pass@2": 0.9428015501968511, + "pass@4": 0.95275325712411, + "pass@8": 0.9602836900431464, + "pass@16": 0.9667643456431163, + "pass@32": 0.9724262544552759, + "pass@64": 0.9773916206622802, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99825, + "pass@2": 0.9988725393700788, + "pass@4": 0.9990494026996625, + "pass@8": 0.9991214535331311, + "pass@16": 0.9992352362176316, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99721875, + "pass@2": 0.9981986958661418, + "pass@4": 0.99854321287964, + "pass@8": 0.9989217378407179, + "pass@16": 0.9993741635872916, + "pass@32": 0.9997590674152546, + "pass@64": 0.9999711690328693, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980671875, + "pass@2": 0.9855286663385826, + "pass@4": 0.9891177830896137, + "pass@8": 0.9923601897966485, + "pass@16": 0.9951406369282718, + "pass@32": 0.9969241199883416, + "pass@64": 0.9980015759808923, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9930234375, + "pass@2": 0.994091658464567, + "pass@4": 0.9947228139295088, + "pass@8": 0.9952883871401448, + "pass@16": 0.9959038226355749, + "pass@32": 0.9965991523224128, + "pass@64": 0.9973491337914664, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9889375, + "pass@2": 0.9931058070866141, + "pass@4": 0.9949000196850394, + "pass@8": 0.9961006060233324, + "pass@16": 0.9972014482074887, + "pass@32": 0.998169276945154, + "pass@64": 0.99912682009508, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9813515625, + "pass@2": 0.9889160925196855, + "pass@4": 0.9925639879077615, + "pass@8": 0.9947117324737054, + "pass@16": 0.9964938152859911, + "pass@32": 0.9980639650979307, + "pass@64": 0.9992073394908063, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9768671875, + "pass@2": 0.9849725639763786, + "pass@4": 0.9882907352830895, + "pass@8": 0.9906114389769584, + "pass@16": 0.9923665586621832, + "pass@32": 0.9938099701806373, + "pass@64": 0.9953752548370095, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9379609375, + "pass@2": 0.9578695866141734, + "pass@4": 0.9690707025684286, + "pass@8": 0.9761228584925753, + "pass@16": 0.9809705725972232, + "pass@32": 0.9846269650875722, + "pass@64": 0.9872749236211029, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.89421875, + "pass@2": 0.9220222687007882, + "pass@4": 0.9362151971316084, + "pass@8": 0.9447333294059417, + "pass@16": 0.9509831248030044, + "pass@32": 0.9562493532460535, + "pass@64": 0.961528048501336, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.8007421875, + "pass@2": 0.850632997047245, + "pass@4": 0.8801703857330323, + "pass@8": 0.900917953033891, + "pass@16": 0.9170591607756168, + "pass@32": 0.9292019660423374, + "pass@64": 0.9391975690385227, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.777703125, + "pass@2": 0.8398607283464579, + "pass@4": 0.8762118897637787, + "pass@8": 0.9005872635175092, + "pass@16": 0.9194266016183247, + "pass@32": 0.9342833957716893, + "pass@64": 0.9452691557195909, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.69453125, + "pass@2": 0.768718626968505, + "pass@4": 0.8212702295650541, + "pass@8": 0.8608277460887901, + "pass@16": 0.8904122978547405, + "pass@32": 0.9119605369492704, + "pass@64": 0.927476651698037, + "pass@128": 0.938 + }, + "16": { + "pass@1": 0.5662421875, + "pass@2": 0.6677634104330711, + "pass@4": 0.7407699299775025, + "pass@8": 0.7935083278076046, + "pass@16": 0.8317948982183991, + "pass@32": 0.8604213110568941, + "pass@64": 0.8841963329743866, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.4808125, + "pass@2": 0.585482406496063, + "pass@4": 0.6681984115110617, + "pass@8": 0.7304857882018998, + "pass@16": 0.7780592856710695, + "pass@32": 0.8142376854501494, + "pass@64": 0.8410238580115026, + "pass@128": 0.864 + }, + "18": { + "pass@1": 0.44084375, + "pass@2": 0.5405799704724411, + "pass@4": 0.6218796379827516, + "pass@8": 0.6872290020729789, + "pass@16": 0.7397756567660226, + "pass@32": 0.7814495225157587, + "pass@64": 0.8142298574628336, + "pass@128": 0.84 + }, + "19": { + "pass@1": 0.2833359375, + "pass@2": 0.37453592519685047, + "pass@4": 0.4594816270153731, + "pass@8": 0.5354858260969473, + "pass@16": 0.6025362039793238, + "pass@32": 0.6610735591981041, + "pass@64": 0.710950493166852, + "pass@128": 0.753 + }, + "20": { + "pass@1": 0.273765625, + "pass@2": 0.37301820866141744, + "pass@4": 0.4676901174540681, + "pass@8": 0.5478492712279519, + "pass@16": 0.6117375768104771, + "pass@32": 0.6630715077173114, + "pass@64": 0.7036115648179885, + "pass@128": 0.736 + } + }, + "per_op_accuracy": { + "10": 0.92821875, + "2": 0.999953125, + "3": 0.99825, + "4": 0.99721875, + "5": 0.980671875, + "6": 0.9930234375, + "7": 0.9889375, + "8": 0.9813515625, + "9": 0.9768671875, + "11": 0.9379609375, + "12": 0.89421875, + "13": 0.8007421875, + "14": 0.777703125, + "15": 0.69453125, + "16": 0.5662421875, + "17": 0.4808125, + "18": 0.44084375, + "19": 0.2833359375, + "20": 0.273765625 + }, + "per_op_avg_response_len": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125, + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125, + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 640313, + "answer_accuracy": 0.790775420882074, + "avg_response_len": 274.5671521794973, + "resp_tokens_sum": 222324711, + "pass_at_k": { + "pass@1": 0.790775420882074, + "pass@2": 0.8310315267138889, + "pass@4": 0.8607817283045475, + "pass@8": 0.8837751849257428, + "pass@16": 0.902361735206419, + "pass@32": 0.9174584369632929, + "pass@64": 0.9296724648662624, + "pass@128": 0.9399304457793234 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9259080014749262, + "pass@2": 0.9433224020393468, + "pass@4": 0.9555353711317055, + "pass@8": 0.9626092113862927, + "pass@16": 0.9672899140408969, + "pass@32": 0.971873515401767, + "pass@64": 0.9768429344531748, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999044342507645, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9946638933316929, + "pass@4": 0.995454402184102, + "pass@8": 0.9966304348689696, + "pass@16": 0.9980442612102862, + "pass@32": 0.9992470856726705, + "pass@64": 0.9999099032277166, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9814980158730159, + "pass@2": 0.9859626921634798, + "pass@4": 0.9891130867570126, + "pass@8": 0.9915862161820935, + "pass@16": 0.9933890901170934, + "pass@32": 0.9943969337435874, + "pass@64": 0.9952380498015997, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.992374625748503, + "pass@2": 0.9937386545806025, + "pass@4": 0.9939952238380383, + "pass@8": 0.9940119126755125, + "pass@16": 0.9940119760476699, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9904539571005917, + "pass@2": 0.9942641114941992, + "pass@4": 0.995715497378656, + "pass@8": 0.9967252188259338, + "pass@16": 0.9975999557269617, + "pass@32": 0.9983364053352927, + "pass@64": 0.9992661788080938, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9777393830128205, + "pass@2": 0.9862255987532803, + "pass@4": 0.990327881490775, + "pass@8": 0.9937928780958446, + "pass@16": 0.996912117506564, + "pass@32": 0.9990931142481682, + "pass@64": 0.9999455717474616, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9797535211267606, + "pass@2": 0.9886772901186649, + "pass@4": 0.9919861029695232, + "pass@8": 0.9939178049019348, + "pass@16": 0.9951668490880456, + "pass@32": 0.996002421656276, + "pass@64": 0.9968393035287623, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9369688385269122, + "pass@2": 0.9556891436729048, + "pass@4": 0.9681532095138248, + "pass@8": 0.9773977476860249, + "pass@16": 0.9836760616393073, + "pass@32": 0.9879154034727532, + "pass@64": 0.9905512293835983, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.896340925141243, + "pass@2": 0.9266628019484855, + "pass@4": 0.9416773099337153, + "pass@8": 0.9499502893000226, + "pass@16": 0.9556055452848286, + "pass@32": 0.9601108862491841, + "pass@64": 0.9653730933928352, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8189410266457681, + "pass@2": 0.8701832590773324, + "pass@4": 0.9008475028003936, + "pass@8": 0.9223142658352539, + "pass@16": 0.9375126996972292, + "pass@32": 0.9461958750908102, + "pass@64": 0.9519937435789704, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8048091900311527, + "pass@2": 0.8633829138764197, + "pass@4": 0.8936096031220395, + "pass@8": 0.9133137052944442, + "pass@16": 0.9308204065370922, + "pass@32": 0.9458697424997475, + "pass@64": 0.9578949685863664, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7142792630057804, + "pass@2": 0.7852296777570432, + "pass@4": 0.8336160666874729, + "pass@8": 0.8703838255639478, + "pass@16": 0.8991492741726431, + "pass@32": 0.9217285219094363, + "pass@64": 0.9390414573776548, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5767389112903226, + "pass@2": 0.671137207899416, + "pass@4": 0.7352816832371761, + "pass@8": 0.7801929643943304, + "pass@16": 0.8153526018049102, + "pass@32": 0.8451755884363006, + "pass@64": 0.8721477928442585, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.49742593930635837, + "pass@2": 0.6042979950844295, + "pass@4": 0.6877961173386561, + "pass@8": 0.750236429513258, + "pass@16": 0.797521632533749, + "pass@32": 0.8336931159590578, + "pass@64": 0.861816202176084, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.4068080357142857, + "pass@2": 0.5067449381327332, + "pass@4": 0.5904474240719905, + "pass@8": 0.6602663145937516, + "pass@16": 0.7184114317717843, + "pass@32": 0.7636522860956406, + "pass@64": 0.7975986551347283, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.28638795045045046, + "pass@2": 0.37785669527795496, + "pass@4": 0.45953530245656216, + "pass@8": 0.5290022873626415, + "pass@16": 0.5902007338149695, + "pass@32": 0.6466124054432371, + "pass@64": 0.6966004194500958, + "pass@128": 0.7417417417417418 + }, + "20": { + "pass@1": 0.2728978978978979, + "pass@2": 0.36531228571090785, + "pass@4": 0.4511123266123264, + "pass@8": 0.5250073329043563, + "pass@16": 0.5869595546727128, + "pass@32": 0.638416275402605, + "pass@64": 0.6777993410094819, + "pass@128": 0.7057057057057057 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 648988, + "answer_accuracy": 0.7903692517537022, + "avg_response_len": 284.4923129384256, + "resp_tokens_sum": 233602328, + "pass_at_k": { + "pass@1": 0.7903692517537022, + "pass@2": 0.8308205308976789, + "pass@4": 0.8615349426362584, + "pass@8": 0.8856540052607581, + "pass@16": 0.9049316985397561, + "pass@32": 0.9202179500084126, + "pass@64": 0.9324577617228758, + "pass@128": 0.9429462197973499 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163391487730062, + "pass@2": 0.9258629564996861, + "pass@4": 0.9336692571212339, + "pass@8": 0.9416640558086454, + "pass@16": 0.9500473404479759, + "pass@32": 0.9578994674031669, + "pass@64": 0.9655855093279391, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9995746092817943, + "pass@4": 0.9999632247673588, + "pass@8": 0.9999996922182559, + "pass@16": 0.9999999999913911, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986131656804734, + "pass@2": 0.999854036597866, + "pass@4": 0.9999976565429319, + "pass@8": 0.9999999993354288, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863224637681159, + "pass@2": 0.9906003937007875, + "pass@4": 0.9930274229308294, + "pass@8": 0.9950581299215392, + "pass@16": 0.9969160592890997, + "pass@32": 0.998278697645461, + "pass@64": 0.9992808085046374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967765042979942, + "pass@2": 0.9973253869322924, + "pass@4": 0.9975613690301607, + "pass@8": 0.9979359856831456, + "pass@16": 0.9985470770655008, + "pass@32": 0.9993380089666443, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9916230130057804, + "pass@2": 0.9936738342587959, + "pass@4": 0.9944413127911034, + "pass@8": 0.9949076292657711, + "pass@16": 0.9955414667011108, + "pass@32": 0.9966227126233496, + "pass@64": 0.9982021755951027, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9815140845070423, + "pass@2": 0.9889029056227125, + "pass@4": 0.9929117410499744, + "pass@8": 0.9949431586627508, + "pass@16": 0.9962341274090126, + "pass@32": 0.9974052544005088, + "pass@64": 0.9985526092972625, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9739427395209581, + "pass@2": 0.9802412890753925, + "pass@4": 0.9834003469439074, + "pass@8": 0.9861000031633753, + "pass@16": 0.9883617473597814, + "pass@32": 0.9900186779076805, + "pass@64": 0.9917721721639942, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.942371046686747, + "pass@2": 0.9614907711554884, + "pass@4": 0.9709818822534232, + "pass@8": 0.9762108984873451, + "pass@16": 0.9798418328465113, + "pass@32": 0.9832802016718227, + "pass@64": 0.9869836840656171, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8921189263322884, + "pass@2": 0.9154603923431991, + "pass@4": 0.9262470747191082, + "pass@8": 0.932674949530915, + "pass@16": 0.9378135612950852, + "pass@32": 0.942572203878569, + "pass@64": 0.947465209542479, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7801995798319328, + "pass@2": 0.8263980789166057, + "pass@4": 0.8551042666410398, + "pass@8": 0.8787522658307619, + "pass@16": 0.8997253159536589, + "pass@32": 0.9175379858563458, + "pass@64": 0.9326565965286922, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7647112573099415, + "pass@2": 0.8316522626283556, + "pass@4": 0.8720206605205054, + "pass@8": 0.899213977549253, + "pass@16": 0.9185417582132726, + "pass@32": 0.9327907985067769, + "pass@64": 0.9426059507605182, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.6902225378787878, + "pass@2": 0.7702934860415173, + "pass@4": 0.8277060339048522, + "pass@8": 0.870181513451221, + "pass@16": 0.9009560991102984, + "pass@32": 0.9213126526056608, + "pass@64": 0.9341966745085428, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5663951210826211, + "pass@2": 0.6729072728088475, + "pass@4": 0.747877350267114, + "pass@8": 0.8014397473398795, + "pass@16": 0.8403655118855234, + "pass@32": 0.8670101699239859, + "pass@64": 0.8852311741120641, + "pass@128": 0.9002849002849003 + }, + "17": { + "pass@1": 0.499708850931677, + "pass@2": 0.6042562020100748, + "pass@4": 0.6857853701905272, + "pass@8": 0.7446535294212936, + "pass@16": 0.787459898111565, + "pass@32": 0.8184923783964188, + "pass@64": 0.840558376080876, + "pass@128": 0.860248447204969 + }, + "18": { + "pass@1": 0.47116268382352944, + "pass@2": 0.5732706837656322, + "pass@4": 0.6578277650036389, + "pass@8": 0.7272127940450731, + "pass@16": 0.7824553129641325, + "pass@32": 0.8258777832780535, + "pass@64": 0.858667717455225, + "pass@128": 0.8823529411764706 + }, + "19": { + "pass@1": 0.2761346726190476, + "pass@2": 0.365666449896888, + "pass@4": 0.45070808615440894, + "pass@8": 0.529863381029973, + "pass@16": 0.6031771625339937, + "pass@32": 0.6680454701301732, + "pass@64": 0.7240389650730391, + "pass@128": 0.7708333333333334 + }, + "20": { + "pass@1": 0.277156624251497, + "pass@2": 0.3758873738742987, + "pass@4": 0.46919662278367874, + "pass@8": 0.547041580122116, + "pass@16": 0.608975135388662, + "pass@32": 0.6595693283084222, + "pass@64": 0.7034790052169011, + "pass@128": 0.7455089820359282 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630014, + "answer_accuracy": 0.7863851054481547, + "avg_response_len": 304.9374862697715, + "resp_tokens_sum": 244301277, + "pass_at_k": { + "pass@1": 0.7863851054481547, + "pass@2": 0.8276325988214681, + "pass@4": 0.8593780909146573, + "pass@8": 0.8839933436948046, + "pass@16": 0.902981023040986, + "pass@32": 0.917834536552863, + "pass@64": 0.9295588405832169, + "pass@128": 0.9389678862438089 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421175373134328, + "pass@2": 0.9587580062286991, + "pass@4": 0.9685092193699665, + "pass@8": 0.9760498065360441, + "pass@16": 0.9825003934245147, + "pass@32": 0.9871221085392369, + "pass@64": 0.9894357905723764, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999330357142857, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9971243629622828, + "pass@4": 0.9972639609917566, + "pass@8": 0.99743893615483, + "pass@16": 0.9977703679897155, + "pass@32": 0.9983643626179382, + "pass@64": 0.9992768761047727, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989720394736842, + "pass@2": 0.9998701334208223, + "pass@4": 0.9999958955240242, + "pass@8": 0.9999999968048913, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.974172794117647, + "pass@2": 0.9799802425891616, + "pass@4": 0.9851549995588786, + "pass@8": 0.9903396378716414, + "pass@16": 0.9949618620195534, + "pass@32": 0.9978909857983171, + "pass@64": 0.9992638569390833, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.98957511829653, + "pass@2": 0.9909034315308379, + "pass@4": 0.9923643261075016, + "pass@8": 0.9937184520602704, + "pass@16": 0.9949870433431968, + "pass@32": 0.9963097387793499, + "pass@64": 0.9980377056438621, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.984375, + "pass@2": 0.9912449074304795, + "pass@4": 0.994530023244721, + "pass@8": 0.9967387415639558, + "pass@16": 0.9985927711809854, + "pass@32": 0.9996839030827411, + "pass@64": 0.9999901863988568, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9845626876876877, + "pass@2": 0.9914509736114068, + "pass@4": 0.9942883507804767, + "pass@8": 0.9953259254730494, + "pass@16": 0.9963787369181493, + "pass@32": 0.9978019463673319, + "pass@64": 0.9992136480632735, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9767132234726688, + "pass@2": 0.9858248727751473, + "pass@4": 0.9893246072335813, + "pass@8": 0.9916823703543539, + "pass@16": 0.9934710727259167, + "pass@32": 0.995379009876187, + "pass@64": 0.9975736224486328, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9344246031746032, + "pass@2": 0.956496453568304, + "pass@4": 0.9680845546092448, + "pass@8": 0.9746013817828894, + "pass@16": 0.979128362963511, + "pass@32": 0.9823612720845869, + "pass@64": 0.9839103382187553, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8939698012232415, + "pass@2": 0.9233999133135886, + "pass@4": 0.940026338164151, + "pass@8": 0.9508489849950189, + "pass@16": 0.9588264089267363, + "pass@32": 0.9654115182769986, + "pass@64": 0.9710843168080171, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8054591049382716, + "pass@2": 0.8580877260134149, + "pass@4": 0.8874315097186924, + "pass@8": 0.9042751337372295, + "pass@16": 0.9160205919035335, + "pass@32": 0.9253222868446103, + "pass@64": 0.9338060490002721, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7650686201780416, + "pass@2": 0.8257855761816872, + "pass@4": 0.8638935942539817, + "pass@8": 0.8898587056265527, + "pass@16": 0.909471720507359, + "pass@32": 0.9247618852817582, + "pass@64": 0.9359455063598505, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6778308256172839, + "pass@2": 0.7494824323174885, + "pass@4": 0.801531108958602, + "pass@8": 0.8410958117430899, + "pass@16": 0.8703430132240998, + "pass@32": 0.8920039907677088, + "pass@64": 0.9082821137578678, + "pass@128": 0.9197530864197531 + }, + "16": { + "pass@1": 0.5564850663716814, + "pass@2": 0.659352280909577, + "pass@4": 0.7384296702956372, + "pass@8": 0.7974724410886839, + "pass@16": 0.8379566282804083, + "pass@32": 0.867540734508324, + "pass@64": 0.8941426993490582, + "pass@128": 0.9174041297935103 + }, + "17": { + "pass@1": 0.44517131024096385, + "pass@2": 0.5476650697277298, + "pass@4": 0.630717065393181, + "pass@8": 0.696161286495951, + "pass@16": 0.7486587759712936, + "pass@32": 0.7898353056767135, + "pass@64": 0.8198061896401653, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.44601814516129035, + "pass@2": 0.5429264827279653, + "pass@4": 0.6179406434074768, + "pass@8": 0.6738175548059356, + "pass@16": 0.7170866104454663, + "pass@32": 0.7528154066701483, + "pass@64": 0.7842687233254886, + "pass@128": 0.8129032258064516 + }, + "19": { + "pass@1": 0.28757552870090636, + "pass@2": 0.3801985453291147, + "pass@4": 0.4683336989409553, + "pass@8": 0.5477159165532208, + "pass@16": 0.6142955679985431, + "pass@32": 0.6685448647183322, + "pass@64": 0.7121010913154943, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2712321696696697, + "pass@2": 0.377846350287295, + "pass@4": 0.48275687892617025, + "pass@8": 0.5715013261561981, + "pass@16": 0.6392863359899448, + "pass@32": 0.6912394364961897, + "pass@64": 0.7295567463044594, + "pass@128": 0.7567567567567568 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..af9288e8d965bc662c8c200abc27a3a5b62ca48a --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bf9380e9142a1e3bf1513ee5fc0db29b04a052b87ae04445cf228f7743116a1 +size 2688868264 diff --git a/results/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv b/results/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d27204e3c928fd0f1c8f556ef8c91a9cad20b681 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op11-14,0.9827213541666666,0.615015625,0.7891920230263157,0.0,0.0,0.0,199.76456336805555,367.26526484375,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch1-op11-14/summary.json b/results/composition/SDAR-100M-block4-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6d3754ec8c3afee4ea27abc4016fc2c6b7d9e498 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op11-14", + "id_acc": 0.9827213541666666, + "ood_acc": 0.615015625, + "total_acc": 0.7891920230263157, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.76456336805555, + "ood_avg_resp_len": 367.26526484375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 640313, + "answer_accuracy": 0.790775420882074, + "avg_response_len": 274.5671521794973, + "resp_tokens_sum": 222324711, + "pass_at_k": { + "pass@1": 0.790775420882074, + "pass@2": 0.8310315267138889, + "pass@4": 0.8607817283045475, + "pass@8": 0.8837751849257428, + "pass@16": 0.902361735206419, + "pass@32": 0.9174584369632929, + "pass@64": 0.9296724648662624, + "pass@128": 0.9399304457793234 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9259080014749262, + "pass@2": 0.9433224020393468, + "pass@4": 0.9555353711317055, + "pass@8": 0.9626092113862927, + "pass@16": 0.9672899140408969, + "pass@32": 0.971873515401767, + "pass@64": 0.9768429344531748, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999044342507645, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9946638933316929, + "pass@4": 0.995454402184102, + "pass@8": 0.9966304348689696, + "pass@16": 0.9980442612102862, + "pass@32": 0.9992470856726705, + "pass@64": 0.9999099032277166, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9814980158730159, + "pass@2": 0.9859626921634798, + "pass@4": 0.9891130867570126, + "pass@8": 0.9915862161820935, + "pass@16": 0.9933890901170934, + "pass@32": 0.9943969337435874, + "pass@64": 0.9952380498015997, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.992374625748503, + "pass@2": 0.9937386545806025, + "pass@4": 0.9939952238380383, + "pass@8": 0.9940119126755125, + "pass@16": 0.9940119760476699, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9904539571005917, + "pass@2": 0.9942641114941992, + "pass@4": 0.995715497378656, + "pass@8": 0.9967252188259338, + "pass@16": 0.9975999557269617, + "pass@32": 0.9983364053352927, + "pass@64": 0.9992661788080938, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9777393830128205, + "pass@2": 0.9862255987532803, + "pass@4": 0.990327881490775, + "pass@8": 0.9937928780958446, + "pass@16": 0.996912117506564, + "pass@32": 0.9990931142481682, + "pass@64": 0.9999455717474616, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9797535211267606, + "pass@2": 0.9886772901186649, + "pass@4": 0.9919861029695232, + "pass@8": 0.9939178049019348, + "pass@16": 0.9951668490880456, + "pass@32": 0.996002421656276, + "pass@64": 0.9968393035287623, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9369688385269122, + "pass@2": 0.9556891436729048, + "pass@4": 0.9681532095138248, + "pass@8": 0.9773977476860249, + "pass@16": 0.9836760616393073, + "pass@32": 0.9879154034727532, + "pass@64": 0.9905512293835983, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.896340925141243, + "pass@2": 0.9266628019484855, + "pass@4": 0.9416773099337153, + "pass@8": 0.9499502893000226, + "pass@16": 0.9556055452848286, + "pass@32": 0.9601108862491841, + "pass@64": 0.9653730933928352, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8189410266457681, + "pass@2": 0.8701832590773324, + "pass@4": 0.9008475028003936, + "pass@8": 0.9223142658352539, + "pass@16": 0.9375126996972292, + "pass@32": 0.9461958750908102, + "pass@64": 0.9519937435789704, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8048091900311527, + "pass@2": 0.8633829138764197, + "pass@4": 0.8936096031220395, + "pass@8": 0.9133137052944442, + "pass@16": 0.9308204065370922, + "pass@32": 0.9458697424997475, + "pass@64": 0.9578949685863664, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7142792630057804, + "pass@2": 0.7852296777570432, + "pass@4": 0.8336160666874729, + "pass@8": 0.8703838255639478, + "pass@16": 0.8991492741726431, + "pass@32": 0.9217285219094363, + "pass@64": 0.9390414573776548, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5767389112903226, + "pass@2": 0.671137207899416, + "pass@4": 0.7352816832371761, + "pass@8": 0.7801929643943304, + "pass@16": 0.8153526018049102, + "pass@32": 0.8451755884363006, + "pass@64": 0.8721477928442585, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.49742593930635837, + "pass@2": 0.6042979950844295, + "pass@4": 0.6877961173386561, + "pass@8": 0.750236429513258, + "pass@16": 0.797521632533749, + "pass@32": 0.8336931159590578, + "pass@64": 0.861816202176084, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.4068080357142857, + "pass@2": 0.5067449381327332, + "pass@4": 0.5904474240719905, + "pass@8": 0.6602663145937516, + "pass@16": 0.7184114317717843, + "pass@32": 0.7636522860956406, + "pass@64": 0.7975986551347283, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.28638795045045046, + "pass@2": 0.37785669527795496, + "pass@4": 0.45953530245656216, + "pass@8": 0.5290022873626415, + "pass@16": 0.5902007338149695, + "pass@32": 0.6466124054432371, + "pass@64": 0.6966004194500958, + "pass@128": 0.7417417417417418 + }, + "20": { + "pass@1": 0.2728978978978979, + "pass@2": 0.36531228571090785, + "pass@4": 0.4511123266123264, + "pass@8": 0.5250073329043563, + "pass@16": 0.5869595546727128, + "pass@32": 0.638416275402605, + "pass@64": 0.6777993410094819, + "pass@128": 0.7057057057057057 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 648988, + "answer_accuracy": 0.7903692517537022, + "avg_response_len": 284.4923129384256, + "resp_tokens_sum": 233602328, + "pass_at_k": { + "pass@1": 0.7903692517537022, + "pass@2": 0.8308205308976789, + "pass@4": 0.8615349426362584, + "pass@8": 0.8856540052607581, + "pass@16": 0.9049316985397561, + "pass@32": 0.9202179500084126, + "pass@64": 0.9324577617228758, + "pass@128": 0.9429462197973499 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163391487730062, + "pass@2": 0.9258629564996861, + "pass@4": 0.9336692571212339, + "pass@8": 0.9416640558086454, + "pass@16": 0.9500473404479759, + "pass@32": 0.9578994674031669, + "pass@64": 0.9655855093279391, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9995746092817943, + "pass@4": 0.9999632247673588, + "pass@8": 0.9999996922182559, + "pass@16": 0.9999999999913911, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986131656804734, + "pass@2": 0.999854036597866, + "pass@4": 0.9999976565429319, + "pass@8": 0.9999999993354288, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863224637681159, + "pass@2": 0.9906003937007875, + "pass@4": 0.9930274229308294, + "pass@8": 0.9950581299215392, + "pass@16": 0.9969160592890997, + "pass@32": 0.998278697645461, + "pass@64": 0.9992808085046374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967765042979942, + "pass@2": 0.9973253869322924, + "pass@4": 0.9975613690301607, + "pass@8": 0.9979359856831456, + "pass@16": 0.9985470770655008, + "pass@32": 0.9993380089666443, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9916230130057804, + "pass@2": 0.9936738342587959, + "pass@4": 0.9944413127911034, + "pass@8": 0.9949076292657711, + "pass@16": 0.9955414667011108, + "pass@32": 0.9966227126233496, + "pass@64": 0.9982021755951027, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9815140845070423, + "pass@2": 0.9889029056227125, + "pass@4": 0.9929117410499744, + "pass@8": 0.9949431586627508, + "pass@16": 0.9962341274090126, + "pass@32": 0.9974052544005088, + "pass@64": 0.9985526092972625, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9739427395209581, + "pass@2": 0.9802412890753925, + "pass@4": 0.9834003469439074, + "pass@8": 0.9861000031633753, + "pass@16": 0.9883617473597814, + "pass@32": 0.9900186779076805, + "pass@64": 0.9917721721639942, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.942371046686747, + "pass@2": 0.9614907711554884, + "pass@4": 0.9709818822534232, + "pass@8": 0.9762108984873451, + "pass@16": 0.9798418328465113, + "pass@32": 0.9832802016718227, + "pass@64": 0.9869836840656171, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8921189263322884, + "pass@2": 0.9154603923431991, + "pass@4": 0.9262470747191082, + "pass@8": 0.932674949530915, + "pass@16": 0.9378135612950852, + "pass@32": 0.942572203878569, + "pass@64": 0.947465209542479, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7801995798319328, + "pass@2": 0.8263980789166057, + "pass@4": 0.8551042666410398, + "pass@8": 0.8787522658307619, + "pass@16": 0.8997253159536589, + "pass@32": 0.9175379858563458, + "pass@64": 0.9326565965286922, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7647112573099415, + "pass@2": 0.8316522626283556, + "pass@4": 0.8720206605205054, + "pass@8": 0.899213977549253, + "pass@16": 0.9185417582132726, + "pass@32": 0.9327907985067769, + "pass@64": 0.9426059507605182, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.6902225378787878, + "pass@2": 0.7702934860415173, + "pass@4": 0.8277060339048522, + "pass@8": 0.870181513451221, + "pass@16": 0.9009560991102984, + "pass@32": 0.9213126526056608, + "pass@64": 0.9341966745085428, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5663951210826211, + "pass@2": 0.6729072728088475, + "pass@4": 0.747877350267114, + "pass@8": 0.8014397473398795, + "pass@16": 0.8403655118855234, + "pass@32": 0.8670101699239859, + "pass@64": 0.8852311741120641, + "pass@128": 0.9002849002849003 + }, + "17": { + "pass@1": 0.499708850931677, + "pass@2": 0.6042562020100748, + "pass@4": 0.6857853701905272, + "pass@8": 0.7446535294212936, + "pass@16": 0.787459898111565, + "pass@32": 0.8184923783964188, + "pass@64": 0.840558376080876, + "pass@128": 0.860248447204969 + }, + "18": { + "pass@1": 0.47116268382352944, + "pass@2": 0.5732706837656322, + "pass@4": 0.6578277650036389, + "pass@8": 0.7272127940450731, + "pass@16": 0.7824553129641325, + "pass@32": 0.8258777832780535, + "pass@64": 0.858667717455225, + "pass@128": 0.8823529411764706 + }, + "19": { + "pass@1": 0.2761346726190476, + "pass@2": 0.365666449896888, + "pass@4": 0.45070808615440894, + "pass@8": 0.529863381029973, + "pass@16": 0.6031771625339937, + "pass@32": 0.6680454701301732, + "pass@64": 0.7240389650730391, + "pass@128": 0.7708333333333334 + }, + "20": { + "pass@1": 0.277156624251497, + "pass@2": 0.3758873738742987, + "pass@4": 0.46919662278367874, + "pass@8": 0.547041580122116, + "pass@16": 0.608975135388662, + "pass@32": 0.6595693283084222, + "pass@64": 0.7034790052169011, + "pass@128": 0.7455089820359282 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630014, + "answer_accuracy": 0.7863851054481547, + "avg_response_len": 304.9374862697715, + "resp_tokens_sum": 244301277, + "pass_at_k": { + "pass@1": 0.7863851054481547, + "pass@2": 0.8276325988214681, + "pass@4": 0.8593780909146573, + "pass@8": 0.8839933436948046, + "pass@16": 0.902981023040986, + "pass@32": 0.917834536552863, + "pass@64": 0.9295588405832169, + "pass@128": 0.9389678862438089 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421175373134328, + "pass@2": 0.9587580062286991, + "pass@4": 0.9685092193699665, + "pass@8": 0.9760498065360441, + "pass@16": 0.9825003934245147, + "pass@32": 0.9871221085392369, + "pass@64": 0.9894357905723764, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999330357142857, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9971243629622828, + "pass@4": 0.9972639609917566, + "pass@8": 0.99743893615483, + "pass@16": 0.9977703679897155, + "pass@32": 0.9983643626179382, + "pass@64": 0.9992768761047727, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989720394736842, + "pass@2": 0.9998701334208223, + "pass@4": 0.9999958955240242, + "pass@8": 0.9999999968048913, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.974172794117647, + "pass@2": 0.9799802425891616, + "pass@4": 0.9851549995588786, + "pass@8": 0.9903396378716414, + "pass@16": 0.9949618620195534, + "pass@32": 0.9978909857983171, + "pass@64": 0.9992638569390833, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.98957511829653, + "pass@2": 0.9909034315308379, + "pass@4": 0.9923643261075016, + "pass@8": 0.9937184520602704, + "pass@16": 0.9949870433431968, + "pass@32": 0.9963097387793499, + "pass@64": 0.9980377056438621, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.984375, + "pass@2": 0.9912449074304795, + "pass@4": 0.994530023244721, + "pass@8": 0.9967387415639558, + "pass@16": 0.9985927711809854, + "pass@32": 0.9996839030827411, + "pass@64": 0.9999901863988568, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9845626876876877, + "pass@2": 0.9914509736114068, + "pass@4": 0.9942883507804767, + "pass@8": 0.9953259254730494, + "pass@16": 0.9963787369181493, + "pass@32": 0.9978019463673319, + "pass@64": 0.9992136480632735, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9767132234726688, + "pass@2": 0.9858248727751473, + "pass@4": 0.9893246072335813, + "pass@8": 0.9916823703543539, + "pass@16": 0.9934710727259167, + "pass@32": 0.995379009876187, + "pass@64": 0.9975736224486328, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9344246031746032, + "pass@2": 0.956496453568304, + "pass@4": 0.9680845546092448, + "pass@8": 0.9746013817828894, + "pass@16": 0.979128362963511, + "pass@32": 0.9823612720845869, + "pass@64": 0.9839103382187553, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8939698012232415, + "pass@2": 0.9233999133135886, + "pass@4": 0.940026338164151, + "pass@8": 0.9508489849950189, + "pass@16": 0.9588264089267363, + "pass@32": 0.9654115182769986, + "pass@64": 0.9710843168080171, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8054591049382716, + "pass@2": 0.8580877260134149, + "pass@4": 0.8874315097186924, + "pass@8": 0.9042751337372295, + "pass@16": 0.9160205919035335, + "pass@32": 0.9253222868446103, + "pass@64": 0.9338060490002721, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7650686201780416, + "pass@2": 0.8257855761816872, + "pass@4": 0.8638935942539817, + "pass@8": 0.8898587056265527, + "pass@16": 0.909471720507359, + "pass@32": 0.9247618852817582, + "pass@64": 0.9359455063598505, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6778308256172839, + "pass@2": 0.7494824323174885, + "pass@4": 0.801531108958602, + "pass@8": 0.8410958117430899, + "pass@16": 0.8703430132240998, + "pass@32": 0.8920039907677088, + "pass@64": 0.9082821137578678, + "pass@128": 0.9197530864197531 + }, + "16": { + "pass@1": 0.5564850663716814, + "pass@2": 0.659352280909577, + "pass@4": 0.7384296702956372, + "pass@8": 0.7974724410886839, + "pass@16": 0.8379566282804083, + "pass@32": 0.867540734508324, + "pass@64": 0.8941426993490582, + "pass@128": 0.9174041297935103 + }, + "17": { + "pass@1": 0.44517131024096385, + "pass@2": 0.5476650697277298, + "pass@4": 0.630717065393181, + "pass@8": 0.696161286495951, + "pass@16": 0.7486587759712936, + "pass@32": 0.7898353056767135, + "pass@64": 0.8198061896401653, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.44601814516129035, + "pass@2": 0.5429264827279653, + "pass@4": 0.6179406434074768, + "pass@8": 0.6738175548059356, + "pass@16": 0.7170866104454663, + "pass@32": 0.7528154066701483, + "pass@64": 0.7842687233254886, + "pass@128": 0.8129032258064516 + }, + "19": { + "pass@1": 0.28757552870090636, + "pass@2": 0.3801985453291147, + "pass@4": 0.4683336989409553, + "pass@8": 0.5477159165532208, + "pass@16": 0.6142955679985431, + "pass@32": 0.6685448647183322, + "pass@64": 0.7121010913154943, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2712321696696697, + "pass@2": 0.377846350287295, + "pass@4": 0.48275687892617025, + "pass@8": 0.5715013261561981, + "pass@16": 0.6392863359899448, + "pass@32": 0.6912394364961897, + "pass@64": 0.7295567463044594, + "pass@128": 0.7567567567567568 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..80dc5af23ad89bea0b045423c7ece1f55c547d6d --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c249d46d93be9a0a91328ad5dc4c638c329356edc18bedf25d6979e61a56eeb +size 1792415605 diff --git a/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json b/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cb719495580e8e4a2cf391c62c1c364b0b583a1b --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9801223958333334, + "avg_response_len": 199.76062586805557, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9801223958333334, + "pass@2": 0.9871357447506511, + "pass@4": 0.9909350812919213, + "pass@8": 0.9933636757087748, + "pass@16": 0.9950767615885424, + "pass@32": 0.9964329006344513, + "pass@64": 0.997616406382146, + "pass@128": 0.9984444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231171875, + "pass@2": 0.9456815944881896, + "pass@4": 0.9603091880389951, + "pass@8": 0.9705846474455383, + "pass@16": 0.9782733645292219, + "pass@32": 0.984408624764542, + "pass@64": 0.9896463722943792, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9976875, + "pass@2": 0.9987242864173229, + "pass@4": 0.9990599458192727, + "pass@8": 0.9991769200116146, + "pass@16": 0.9993323490285145, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9980625, + "pass@2": 0.9988898868110238, + "pass@4": 0.9991574159167605, + "pass@8": 0.9993262463197925, + "pass@16": 0.9995588902104531, + "pass@32": 0.9998290717623712, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.978015625, + "pass@2": 0.9848229576771658, + "pass@4": 0.988770462129734, + "pass@8": 0.991504906288707, + "pass@16": 0.993650861087387, + "pass@32": 0.9955520574897151, + "pass@64": 0.9970482140193007, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9920078125, + "pass@2": 0.9939662893700788, + "pass@4": 0.9950509417885265, + "pass@8": 0.9958147741453353, + "pass@16": 0.9964508524538155, + "pass@32": 0.9971523647058177, + "pass@64": 0.99794013537104, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9861875, + "pass@2": 0.9924363927165356, + "pass@4": 0.9952590014998124, + "pass@8": 0.9965242680443928, + "pass@16": 0.9971608534876047, + "pass@32": 0.9976626144839646, + "pass@64": 0.9982518300815967, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.97428125, + "pass@2": 0.9852411417322838, + "pass@4": 0.99118949015748, + "pass@8": 0.9948321772055063, + "pass@16": 0.9971259158175276, + "pass@32": 0.998483992181002, + "pass@64": 0.9994439058232791, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.971796875, + "pass@2": 0.9844603838582684, + "pass@4": 0.9896192867454066, + "pass@8": 0.9925091419180848, + "pass@16": 0.9941377676823531, + "pass@32": 0.9952259217549699, + "pass@64": 0.9963530707926768, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9231171875, + "2": 0.9999453125, + "3": 0.9976875, + "4": 0.9980625, + "5": 0.978015625, + "6": 0.9920078125, + "7": 0.9861875, + "8": 0.97428125, + "9": 0.971796875 + }, + "per_op_avg_response_len": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371099, + "answer_accuracy": 0.979132366599122, + "avg_response_len": 190.26763287318474, + "resp_tokens_sum": 72112955, + "pass_at_k": { + "pass@1": 0.979132366599122, + "pass@2": 0.9861054602217292, + "pass@4": 0.9899340251224931, + "pass@8": 0.9924987218896144, + "pass@16": 0.9943021555612757, + "pass@32": 0.9957042334254549, + "pass@64": 0.9968508765699006, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9192477876106194, + "pass@2": 0.9396735564304464, + "pass@4": 0.9543033865788902, + "pass@8": 0.96485112476411, + "pass@16": 0.9724661046772268, + "pass@32": 0.9786862175801799, + "pass@64": 0.9841720997110378, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9998296339563862, + "pass@2": 0.9999961672431132, + "pass@4": 0.9999999985399022, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.999965009391028, + "pass@4": 0.9999997130519236, + "pass@8": 0.9999999999935767, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.996728515625, + "pass@4": 0.9973700640935508, + "pass@8": 0.9978945203621713, + "pass@16": 0.9986215319076661, + "pass@32": 0.99946584925741, + "pass@64": 0.999956800941041, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9784474206349206, + "pass@2": 0.9849374296962881, + "pass@4": 0.9890754283690729, + "pass@8": 0.9921342884438079, + "pass@16": 0.9943983611486136, + "pass@32": 0.9960648708043484, + "pass@64": 0.9967769255677263, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9922342814371258, + "pass@2": 0.9939438298599651, + "pass@4": 0.9941038252516338, + "pass@8": 0.9941991013702666, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9887435281065089, + "pass@2": 0.9931025922517821, + "pass@4": 0.995356032973689, + "pass@8": 0.99687459097789, + "pass@16": 0.997698902492233, + "pass@32": 0.9983400676934957, + "pass@64": 0.9992661790057259, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9668469551282052, + "pass@2": 0.9809139505097918, + "pass@4": 0.9884821623859518, + "pass@8": 0.9934550534186496, + "pass@16": 0.9968600221493318, + "pass@32": 0.9988651869448563, + "pass@64": 0.9998085724581219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9736575704225352, + "pass@2": 0.987104566374626, + "pass@4": 0.9918345429532572, + "pass@8": 0.9940607002019898, + "pass@16": 0.9952133611440159, + "pass@32": 0.9960034370145185, + "pass@64": 0.9968393035368946, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383005, + "answer_accuracy": 0.980414994266055, + "avg_response_len": 197.6951153956422, + "resp_tokens_sum": 77230783, + "pass_at_k": { + "pass@1": 0.980414994266055, + "pass@2": 0.9862497387797863, + "pass@4": 0.9896156322837595, + "pass@8": 0.9920707717579302, + "pass@16": 0.993936643245879, + "pass@32": 0.9954022931445885, + "pass@64": 0.9968295599606264, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9164350076687117, + "pass@2": 0.9361730713492108, + "pass@4": 0.9499516876916461, + "pass@8": 0.9618509716297367, + "pass@16": 0.9718630754382047, + "pass@32": 0.9796931696551145, + "pass@64": 0.986608430448021, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973958333333334, + "pass@2": 0.9992700131233596, + "pass@4": 0.9999176963674994, + "pass@8": 0.9999987979293221, + "pass@16": 0.9999999998399193, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9998897084517542, + "pass@4": 0.9999981377142946, + "pass@8": 0.9999999995762411, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980570652173913, + "pass@2": 0.9865802949903003, + "pass@4": 0.9889661506985535, + "pass@8": 0.9902841199998196, + "pass@16": 0.9917663843037684, + "pass@32": 0.9936594509266741, + "pass@64": 0.9954604899202021, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.995008058739255, + "pass@2": 0.9963252713038379, + "pass@4": 0.9970167348350797, + "pass@8": 0.9973006814242409, + "pass@16": 0.997492802179556, + "pass@32": 0.9978510028653035, + "pass@64": 0.998567335243553, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9898843930635838, + "pass@2": 0.9931856196804879, + "pass@4": 0.9941058232063475, + "pass@8": 0.9943886704848257, + "pass@16": 0.9945808983691612, + "pass@32": 0.9949421965317783, + "pass@64": 0.9956647398843931, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9714788732394366, + "pass@2": 0.980618207275147, + "pass@4": 0.9870130438800786, + "pass@8": 0.9915782021625016, + "pass@16": 0.9946980159453276, + "pass@32": 0.9967270976372445, + "pass@64": 0.9986017780742946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9722352170658682, + "pass@2": 0.9828459468857563, + "pass@4": 0.9884671896364751, + "pass@8": 0.9924020194851084, + "pass@16": 0.9944314239658919, + "pass@32": 0.9953209435293606, + "pass@64": 0.9962633785477046, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 374997, + "answer_accuracy": 0.9808048418145296, + "avg_response_len": 211.28144616253766, + "resp_tokens_sum": 80780503, + "pass_at_k": { + "pass@1": 0.9808048418145296, + "pass@2": 0.9890623476007588, + "pass@4": 0.993275585373138, + "pass@8": 0.9955421394237094, + "pass@16": 0.9970095535632802, + "pass@32": 0.9982082597455659, + "pass@64": 0.9991792416859733, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.933535447761194, + "pass@2": 0.9610144405923143, + "pass@4": 0.9764659397052976, + "pass@8": 0.9848856698483921, + "pass@16": 0.9903880371665308, + "pass@32": 0.994788130439683, + "pass@64": 0.9981423348247224, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9970163965473705, + "pass@4": 0.997338775072941, + "pass@8": 0.9976015064053606, + "pass@16": 0.9980534958639686, + "pass@32": 0.9987797625879855, + "pass@64": 0.9996441771309199, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.999924094603306, + "pass@4": 0.9999989030976391, + "pass@8": 0.999999999845404, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9750229779411764, + "pass@2": 0.9829337222093564, + "pass@4": 0.9882893535366903, + "pass@8": 0.9921605412616765, + "pass@16": 0.9948705139434515, + "pass@32": 0.9969973900489489, + "pass@64": 0.998910627773521, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9884660883280757, + "pass@2": 0.9913928407312652, + "pass@4": 0.9938846173408135, + "pass@8": 0.9958811876044368, + "pass@16": 0.9974790678017365, + "pass@32": 0.9989033586934598, + "pass@64": 0.9998111525900316, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9794056566455697, + "pass@2": 0.9909034560948872, + "pass@4": 0.9964178719155359, + "pass@8": 0.9984878997029001, + "pass@16": 0.9994102329414561, + "pass@32": 0.9999166822897716, + "pass@64": 0.9999995619546246, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842342342342343, + "pass@2": 0.9942238005722256, + "pass@4": 0.9981784532001067, + "pass@8": 0.9995914077213195, + "pass@16": 0.9999633430821163, + "pass@32": 0.9999998023813371, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9692021704180064, + "pass@2": 0.983175937412968, + "pass@4": 0.9883279185158123, + "pass@8": 0.9908531152358594, + "pass@16": 0.9925946265968474, + "pass@32": 0.9942363552283263, + "pass@64": 0.9958943717110796, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.62273671875, + "avg_response_len": 372.483859375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.62273671875, + "pass@2": 0.7036788877952637, + "pass@4": 0.7644948062710833, + "pass@8": 0.8099061159706736, + "pass@16": 0.8439089202647532, + "pass@32": 0.8701182617095065, + "pass@64": 0.891005192451688, + "pass@128": 0.9073 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9151171875, + "pass@2": 0.9456545275590561, + "pass@4": 0.9612200796775406, + "pass@8": 0.9704336964892437, + "pass@16": 0.9765719952540661, + "pass@32": 0.9812601474160046, + "pass@64": 0.9858234208718162, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8489609375, + "pass@2": 0.8991227854330703, + "pass@4": 0.9283502003187095, + "pass@8": 0.9455536075739712, + "pass@16": 0.9560643833010687, + "pass@32": 0.9634465601168278, + "pass@64": 0.9699825720446108, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.7550703125, + "pass@2": 0.8208409202755905, + "pass@4": 0.8626456775403079, + "pass@8": 0.890856563637159, + "pass@16": 0.9113103719914247, + "pass@32": 0.9282444428549655, + "pass@64": 0.9434633808607322, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.7476953125, + "pass@2": 0.8176723671259848, + "pass@4": 0.8618266462317206, + "pass@8": 0.8918874643176163, + "pass@16": 0.9141369979757303, + "pass@32": 0.9316738111035253, + "pass@64": 0.9455826385211541, + "pass@128": 0.956 + }, + "15": { + "pass@1": 0.688828125, + "pass@2": 0.7694915108267728, + "pass@4": 0.8278998459880011, + "pass@8": 0.8700518416925825, + "pass@16": 0.8990736716148421, + "pass@32": 0.9185506095706097, + "pass@64": 0.9329091553818495, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.569984375, + "pass@2": 0.6710280511811033, + "pass@4": 0.7460000776152982, + "pass@8": 0.8009541520148893, + "pass@16": 0.8402356139913965, + "pass@32": 0.8684268111847793, + "pass@64": 0.8891133101235712, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.5203671875, + "pass@2": 0.6227608267716539, + "pass@4": 0.7001498685789274, + "pass@8": 0.7556166147417005, + "pass@16": 0.7964709061393579, + "pass@32": 0.8294317709926918, + "pass@64": 0.8556712149259554, + "pass@128": 0.873 + }, + "18": { + "pass@1": 0.4865859375, + "pass@2": 0.5871370570866156, + "pass@4": 0.6677094760030015, + "pass@8": 0.7302382482448312, + "pass@16": 0.777847828734549, + "pass@32": 0.8141039437356614, + "pass@64": 0.8426591360477864, + "pass@128": 0.867 + }, + "19": { + "pass@1": 0.3416015625, + "pass@2": 0.4410281742125985, + "pass@4": 0.5310940967379066, + "pass@8": 0.6078161499766409, + "pass@16": 0.672866477645552, + "pass@32": 0.728375162906233, + "pass@64": 0.7747117439388957, + "pass@128": 0.812 + }, + "20": { + "pass@1": 0.35315625, + "pass@2": 0.4620526574803147, + "pass@4": 0.5580520940194974, + "pass@8": 0.6356528210181952, + "pass@16": 0.6945109559994428, + "pass@32": 0.7376693572137135, + "pass@64": 0.7701353518004574, + "pass@128": 0.794 + } + }, + "per_op_accuracy": { + "11": 0.9151171875, + "12": 0.8489609375, + "13": 0.7550703125, + "14": 0.7476953125, + "15": 0.688828125, + "16": 0.569984375, + "17": 0.5203671875, + "18": 0.4865859375, + "19": 0.3416015625, + "20": 0.35315625 + }, + "per_op_avg_response_len": { + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 268987, + "answer_accuracy": 0.6245054791976226, + "avg_response_len": 354.4513581909361, + "resp_tokens_sum": 152669289, + "pass_at_k": { + "pass@1": 0.6245054791976226, + "pass@2": 0.7043694279346253, + "pass@4": 0.7637833167306537, + "pass@8": 0.8083928321539314, + "pass@16": 0.842271539248331, + "pass@32": 0.8685320453378127, + "pass@64": 0.889781440443194, + "pass@128": 0.9069836552748886 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9049442280453258, + "pass@2": 0.9343667049586221, + "pass@4": 0.9518926340829211, + "pass@8": 0.963721513224062, + "pass@16": 0.9715522567392245, + "pass@32": 0.9772850487323939, + "pass@64": 0.9837385106596693, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8367319915254238, + "pass@2": 0.8906847780150366, + "pass@4": 0.9210060177859121, + "pass@8": 0.9395740180895393, + "pass@16": 0.9513328654341852, + "pass@32": 0.9589354102848824, + "pass@64": 0.9654301877762941, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7604574843260188, + "pass@2": 0.82680597894503, + "pass@4": 0.8719512272956471, + "pass@8": 0.9045499773982028, + "pass@16": 0.9266465799635933, + "pass@32": 0.9410437423571842, + "pass@64": 0.9509158308973618, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.7687889408099688, + "pass@2": 0.8359775523094659, + "pass@4": 0.8736571856672118, + "pass@8": 0.8984409718591754, + "pass@16": 0.9181805371311775, + "pass@32": 0.9350351705266242, + "pass@64": 0.9502913100353946, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.6999638728323699, + "pass@2": 0.7756271761640345, + "pass@4": 0.8292882124380401, + "pass@8": 0.869390789970966, + "pass@16": 0.8979501614129438, + "pass@32": 0.9183570437604774, + "pass@64": 0.9343777336335435, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5773941532258065, + "pass@2": 0.6756885795021594, + "pass@4": 0.7475061202027162, + "pass@8": 0.7991479896927378, + "pass@16": 0.8357307006684896, + "pass@32": 0.8628873839423076, + "pass@64": 0.8835885830672884, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5140895953757225, + "pass@2": 0.6143314624960177, + "pass@4": 0.6916131304576808, + "pass@8": 0.7469650535201684, + "pass@16": 0.7860427028084981, + "pass@32": 0.8170511372956204, + "pass@64": 0.8421452842507103, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.4582589285714286, + "pass@2": 0.5605532902137236, + "pass@4": 0.6421960956666121, + "pass@8": 0.7057777001868978, + "pass@16": 0.7555140844320166, + "pass@32": 0.7930716900033024, + "pass@64": 0.822862301855235, + "pass@128": 0.8514285714285714 + }, + "19": { + "pass@1": 0.3429288663663664, + "pass@2": 0.44343987195857243, + "pass@4": 0.531017780998096, + "pass@8": 0.602912347397304, + "pass@16": 0.6667933588207599, + "pass@32": 0.725113626356638, + "pass@64": 0.775691031987435, + "pass@128": 0.8168168168168168 + }, + "20": { + "pass@1": 0.36878284534534533, + "pass@2": 0.4766399026979738, + "pass@4": 0.570342090403114, + "pass@8": 0.6477507011049768, + "pass@16": 0.7086113239762887, + "pass@32": 0.7532795279937846, + "pass@64": 0.7862179025168473, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 268409, + "answer_accuracy": 0.6235341399048469, + "avg_response_len": 367.8219200676479, + "resp_tokens_sum": 158334095, + "pass_at_k": { + "pass@1": 0.6235341399048469, + "pass@2": 0.7053542370540048, + "pass@4": 0.7665724703123105, + "pass@8": 0.8122417615531752, + "pass@16": 0.8466933293355862, + "pass@32": 0.8734423092418954, + "pass@64": 0.8947083272749418, + "pass@128": 0.9107939339875112 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9257341867469879, + "pass@2": 0.9553495725026092, + "pass@4": 0.9682735500494665, + "pass@8": 0.9742548441493746, + "pass@16": 0.978185160269226, + "pass@32": 0.9822869779847758, + "pass@64": 0.9871868036674503, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8492358934169278, + "pass@2": 0.8949850973761508, + "pass@4": 0.9207448059000464, + "pass@8": 0.9354093985604018, + "pass@16": 0.943759425744798, + "pass@32": 0.9496060392426146, + "pass@64": 0.9561026755224603, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.740983893557423, + "pass@2": 0.8052852538653259, + "pass@4": 0.845815909040781, + "pass@8": 0.8738550513505369, + "pass@16": 0.895515341616072, + "pass@32": 0.9155451542423412, + "pass@64": 0.9353134265154689, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7266081871345029, + "pass@2": 0.7990618668554583, + "pass@4": 0.8478731022109073, + "pass@8": 0.8837960270998939, + "pass@16": 0.9105606572428289, + "pass@32": 0.930775272521883, + "pass@64": 0.9463526228698942, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6870501893939394, + "pass@2": 0.7726907361011697, + "pass@4": 0.8339367397257161, + "pass@8": 0.8770493599523325, + "pass@16": 0.9072087108288012, + "pass@32": 0.9277764122574453, + "pass@64": 0.9424641653105227, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5663728632478633, + "pass@2": 0.668794655876349, + "pass@4": 0.74511328258113, + "pass@8": 0.8007343935830623, + "pass@16": 0.8421667071713081, + "pass@32": 0.8733083123762971, + "pass@64": 0.8957480080042212, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5376067546583851, + "pass@2": 0.6423940553626447, + "pass@4": 0.7202086238637875, + "pass@8": 0.7743041381948508, + "pass@16": 0.814345713501841, + "pass@32": 0.8470088033255796, + "pass@64": 0.8726913345902738, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5092830882352941, + "pass@2": 0.6142293162343674, + "pass@4": 0.7005960897351063, + "pass@8": 0.7701591745990866, + "pass@16": 0.8221200586922632, + "pass@32": 0.8587938588657354, + "pass@64": 0.8857339251123276, + "pass@128": 0.9058823529411765 + }, + "19": { + "pass@1": 0.34770275297619047, + "pass@2": 0.44574237790588667, + "pass@4": 0.5338633851014157, + "pass@8": 0.611502029947303, + "pass@16": 0.6783513518981501, + "pass@32": 0.7327937015644227, + "pass@64": 0.7742047877222114, + "pass@128": 0.8035714285714286 + }, + "20": { + "pass@1": 0.35043974550898205, + "pass@2": 0.4594935375312366, + "pass@4": 0.5528386059714089, + "pass@8": 0.6238549348742458, + "pass@16": 0.6762174612820911, + "pass@32": 0.7171414832463683, + "pass@64": 0.7511741428622221, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 259707, + "answer_accuracy": 0.6200980860330073, + "avg_response_len": 395.8204939639364, + "resp_tokens_sum": 165775956, + "pass_at_k": { + "pass@1": 0.6200980860330073, + "pass@2": 0.7012467768154549, + "pass@4": 0.7630910709816561, + "pass@8": 0.8090618078867852, + "pass@16": 0.8427309922131079, + "pass@32": 0.8683330680173509, + "pass@64": 0.8884576017114336, + "pass@128": 0.9040342298288508 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.915327380952381, + "pass@2": 0.948085786151731, + "pass@4": 0.9642386070788763, + "pass@8": 0.9739282351224063, + "pass@16": 0.9804970648118635, + "pass@32": 0.9846325349921398, + "pass@64": 0.986722881877331, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8619313837920489, + "pass@2": 0.912293969274483, + "pass@4": 0.9437201129675306, + "pass@8": 0.9619229572767756, + "pass@16": 0.9731904957944275, + "pass@32": 0.9818321050690701, + "pass@64": 0.9884511684407877, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7652874228395061, + "pass@2": 0.8321079548216199, + "pass@4": 0.8720276435352986, + "pass@8": 0.896107615787007, + "pass@16": 0.9136146173645132, + "pass@32": 0.929635398081815, + "pass@64": 0.9451059800569461, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7490031528189911, + "pass@2": 0.8191228913058717, + "pass@4": 0.864718364024971, + "pass@8": 0.8938565907497251, + "pass@16": 0.9139148391085333, + "pass@32": 0.9293839114599252, + "pass@64": 0.9403161156625768, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6787471064814815, + "pass@2": 0.7596807560513266, + "pass@4": 0.8202685196757814, + "pass@8": 0.8636306777109823, + "pass@16": 0.8919877814582083, + "pass@32": 0.9093606678534814, + "pass@64": 0.9216089042968235, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.5669478244837758, + "pass@2": 0.6690786646691279, + "pass@4": 0.7455410565382864, + "pass@8": 0.8028333423672143, + "pass@16": 0.8423557007877164, + "pass@32": 0.8684380664265011, + "pass@64": 0.8872958659682277, + "pass@128": 0.9026548672566371 + }, + "17": { + "pass@1": 0.5101891942771084, + "pass@2": 0.6125037798833123, + "pass@4": 0.6895920137241878, + "pass@8": 0.7465083545933107, + "pass@16": 0.7900024434338143, + "pass@32": 0.8252868759565376, + "pass@64": 0.85326005071428, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4936743951612903, + "pass@2": 0.5874368967487934, + "pass@4": 0.6604457161604794, + "pass@8": 0.7140707542442847, + "pass@16": 0.7545067071870171, + "pass@32": 0.7888352910327632, + "pass@64": 0.8177670834201991, + "pass@128": 0.8419354838709677 + }, + "19": { + "pass@1": 0.3340728851963746, + "pass@2": 0.4338164890453644, + "pass@4": 0.5283597530860003, + "pass@8": 0.6090080248672055, + "pass@16": 0.6734085647143847, + "pass@32": 0.7271711226701715, + "pass@64": 0.7742411529076022, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.34025431681681684, + "pass@2": 0.45003221725662673, + "pass@4": 0.5509912418042338, + "pass@8": 0.6353882561628813, + "pass@16": 0.6987590181595222, + "pass@32": 0.7426487056681589, + "pass@64": 0.7730709505896831, + "pass@128": 0.7957957957957958 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7920246710526315, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7920246710526315, + "pass@2": 0.837947925300431, + "pass@4": 0.871755989175678, + "pass@8": 0.8968070653203173, + "pass@16": 0.9155147398391692, + "pass@32": 0.9299515117265957, + "pass@64": 0.9415052411555864, + "pass@128": 0.9504736842105264 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231171875, + "pass@2": 0.9456815944881896, + "pass@4": 0.9603091880389951, + "pass@8": 0.9705846474455383, + "pass@16": 0.9782733645292219, + "pass@32": 0.984408624764542, + "pass@64": 0.9896463722943792, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9976875, + "pass@2": 0.9987242864173229, + "pass@4": 0.9990599458192727, + "pass@8": 0.9991769200116146, + "pass@16": 0.9993323490285145, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9980625, + "pass@2": 0.9988898868110238, + "pass@4": 0.9991574159167605, + "pass@8": 0.9993262463197925, + "pass@16": 0.9995588902104531, + "pass@32": 0.9998290717623712, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.978015625, + "pass@2": 0.9848229576771658, + "pass@4": 0.988770462129734, + "pass@8": 0.991504906288707, + "pass@16": 0.993650861087387, + "pass@32": 0.9955520574897151, + "pass@64": 0.9970482140193007, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9920078125, + "pass@2": 0.9939662893700788, + "pass@4": 0.9950509417885265, + "pass@8": 0.9958147741453353, + "pass@16": 0.9964508524538155, + "pass@32": 0.9971523647058177, + "pass@64": 0.99794013537104, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9861875, + "pass@2": 0.9924363927165356, + "pass@4": 0.9952590014998124, + "pass@8": 0.9965242680443928, + "pass@16": 0.9971608534876047, + "pass@32": 0.9976626144839646, + "pass@64": 0.9982518300815967, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.97428125, + "pass@2": 0.9852411417322838, + "pass@4": 0.99118949015748, + "pass@8": 0.9948321772055063, + "pass@16": 0.9971259158175276, + "pass@32": 0.998483992181002, + "pass@64": 0.9994439058232791, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.971796875, + "pass@2": 0.9844603838582684, + "pass@4": 0.9896192867454066, + "pass@8": 0.9925091419180848, + "pass@16": 0.9941377676823531, + "pass@32": 0.9952259217549699, + "pass@64": 0.9963530707926768, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9151171875, + "pass@2": 0.9456545275590561, + "pass@4": 0.9612200796775406, + "pass@8": 0.9704336964892437, + "pass@16": 0.9765719952540661, + "pass@32": 0.9812601474160046, + "pass@64": 0.9858234208718162, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8489609375, + "pass@2": 0.8991227854330703, + "pass@4": 0.9283502003187095, + "pass@8": 0.9455536075739712, + "pass@16": 0.9560643833010687, + "pass@32": 0.9634465601168278, + "pass@64": 0.9699825720446108, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.7550703125, + "pass@2": 0.8208409202755905, + "pass@4": 0.8626456775403079, + "pass@8": 0.890856563637159, + "pass@16": 0.9113103719914247, + "pass@32": 0.9282444428549655, + "pass@64": 0.9434633808607322, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.7476953125, + "pass@2": 0.8176723671259848, + "pass@4": 0.8618266462317206, + "pass@8": 0.8918874643176163, + "pass@16": 0.9141369979757303, + "pass@32": 0.9316738111035253, + "pass@64": 0.9455826385211541, + "pass@128": 0.956 + }, + "15": { + "pass@1": 0.688828125, + "pass@2": 0.7694915108267728, + "pass@4": 0.8278998459880011, + "pass@8": 0.8700518416925825, + "pass@16": 0.8990736716148421, + "pass@32": 0.9185506095706097, + "pass@64": 0.9329091553818495, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.569984375, + "pass@2": 0.6710280511811033, + "pass@4": 0.7460000776152982, + "pass@8": 0.8009541520148893, + "pass@16": 0.8402356139913965, + "pass@32": 0.8684268111847793, + "pass@64": 0.8891133101235712, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.5203671875, + "pass@2": 0.6227608267716539, + "pass@4": 0.7001498685789274, + "pass@8": 0.7556166147417005, + "pass@16": 0.7964709061393579, + "pass@32": 0.8294317709926918, + "pass@64": 0.8556712149259554, + "pass@128": 0.873 + }, + "18": { + "pass@1": 0.4865859375, + "pass@2": 0.5871370570866156, + "pass@4": 0.6677094760030015, + "pass@8": 0.7302382482448312, + "pass@16": 0.777847828734549, + "pass@32": 0.8141039437356614, + "pass@64": 0.8426591360477864, + "pass@128": 0.867 + }, + "19": { + "pass@1": 0.3416015625, + "pass@2": 0.4410281742125985, + "pass@4": 0.5310940967379066, + "pass@8": 0.6078161499766409, + "pass@16": 0.672866477645552, + "pass@32": 0.728375162906233, + "pass@64": 0.7747117439388957, + "pass@128": 0.812 + }, + "20": { + "pass@1": 0.35315625, + "pass@2": 0.4620526574803147, + "pass@4": 0.5580520940194974, + "pass@8": 0.6356528210181952, + "pass@16": 0.6945109559994428, + "pass@32": 0.7376693572137135, + "pass@64": 0.7701353518004574, + "pass@128": 0.794 + } + }, + "per_op_accuracy": { + "10": 0.9231171875, + "2": 0.9999453125, + "3": 0.9976875, + "4": 0.9980625, + "5": 0.978015625, + "6": 0.9920078125, + "7": 0.9861875, + "8": 0.97428125, + "9": 0.971796875, + "11": 0.9151171875, + "12": 0.8489609375, + "13": 0.7550703125, + "14": 0.7476953125, + "15": 0.688828125, + "16": 0.569984375, + "17": 0.5203671875, + "18": 0.4865859375, + "19": 0.3416015625, + "20": 0.35315625 + }, + "per_op_avg_response_len": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875, + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875, + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 640086, + "answer_accuracy": 0.790495079829276, + "avg_response_len": 277.60216270154916, + "resp_tokens_sum": 224782244, + "pass_at_k": { + "pass@1": 0.790495079829276, + "pass@2": 0.8362411306855028, + "pass@4": 0.8696372919991009, + "pass@8": 0.8945669610675093, + "pass@16": 0.913432249792536, + "pass@32": 0.9280573139004922, + "pass@64": 0.9398972482792994, + "pass@128": 0.9494151122352197 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9192477876106194, + "pass@2": 0.9396735564304464, + "pass@4": 0.9543033865788902, + "pass@8": 0.96485112476411, + "pass@16": 0.9724661046772268, + "pass@32": 0.9786862175801799, + "pass@64": 0.9841720997110378, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9998296339563862, + "pass@2": 0.9999961672431132, + "pass@4": 0.9999999985399022, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.999965009391028, + "pass@4": 0.9999997130519236, + "pass@8": 0.9999999999935767, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.996728515625, + "pass@4": 0.9973700640935508, + "pass@8": 0.9978945203621713, + "pass@16": 0.9986215319076661, + "pass@32": 0.99946584925741, + "pass@64": 0.999956800941041, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9784474206349206, + "pass@2": 0.9849374296962881, + "pass@4": 0.9890754283690729, + "pass@8": 0.9921342884438079, + "pass@16": 0.9943983611486136, + "pass@32": 0.9960648708043484, + "pass@64": 0.9967769255677263, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9922342814371258, + "pass@2": 0.9939438298599651, + "pass@4": 0.9941038252516338, + "pass@8": 0.9941991013702666, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9887435281065089, + "pass@2": 0.9931025922517821, + "pass@4": 0.995356032973689, + "pass@8": 0.99687459097789, + "pass@16": 0.997698902492233, + "pass@32": 0.9983400676934957, + "pass@64": 0.9992661790057259, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9668469551282052, + "pass@2": 0.9809139505097918, + "pass@4": 0.9884821623859518, + "pass@8": 0.9934550534186496, + "pass@16": 0.9968600221493318, + "pass@32": 0.9988651869448563, + "pass@64": 0.9998085724581219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9736575704225352, + "pass@2": 0.987104566374626, + "pass@4": 0.9918345429532572, + "pass@8": 0.9940607002019898, + "pass@16": 0.9952133611440159, + "pass@32": 0.9960034370145185, + "pass@64": 0.9968393035368946, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9049442280453258, + "pass@2": 0.9343667049586221, + "pass@4": 0.9518926340829211, + "pass@8": 0.963721513224062, + "pass@16": 0.9715522567392245, + "pass@32": 0.9772850487323939, + "pass@64": 0.9837385106596693, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8367319915254238, + "pass@2": 0.8906847780150366, + "pass@4": 0.9210060177859121, + "pass@8": 0.9395740180895393, + "pass@16": 0.9513328654341852, + "pass@32": 0.9589354102848824, + "pass@64": 0.9654301877762941, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7604574843260188, + "pass@2": 0.82680597894503, + "pass@4": 0.8719512272956471, + "pass@8": 0.9045499773982028, + "pass@16": 0.9266465799635933, + "pass@32": 0.9410437423571842, + "pass@64": 0.9509158308973618, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.7687889408099688, + "pass@2": 0.8359775523094659, + "pass@4": 0.8736571856672118, + "pass@8": 0.8984409718591754, + "pass@16": 0.9181805371311775, + "pass@32": 0.9350351705266242, + "pass@64": 0.9502913100353946, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.6999638728323699, + "pass@2": 0.7756271761640345, + "pass@4": 0.8292882124380401, + "pass@8": 0.869390789970966, + "pass@16": 0.8979501614129438, + "pass@32": 0.9183570437604774, + "pass@64": 0.9343777336335435, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5773941532258065, + "pass@2": 0.6756885795021594, + "pass@4": 0.7475061202027162, + "pass@8": 0.7991479896927378, + "pass@16": 0.8357307006684896, + "pass@32": 0.8628873839423076, + "pass@64": 0.8835885830672884, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5140895953757225, + "pass@2": 0.6143314624960177, + "pass@4": 0.6916131304576808, + "pass@8": 0.7469650535201684, + "pass@16": 0.7860427028084981, + "pass@32": 0.8170511372956204, + "pass@64": 0.8421452842507103, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.4582589285714286, + "pass@2": 0.5605532902137236, + "pass@4": 0.6421960956666121, + "pass@8": 0.7057777001868978, + "pass@16": 0.7555140844320166, + "pass@32": 0.7930716900033024, + "pass@64": 0.822862301855235, + "pass@128": 0.8514285714285714 + }, + "19": { + "pass@1": 0.3429288663663664, + "pass@2": 0.44343987195857243, + "pass@4": 0.531017780998096, + "pass@8": 0.602912347397304, + "pass@16": 0.6667933588207599, + "pass@32": 0.725113626356638, + "pass@64": 0.775691031987435, + "pass@128": 0.8168168168168168 + }, + "20": { + "pass@1": 0.36878284534534533, + "pass@2": 0.4766399026979738, + "pass@4": 0.570342090403114, + "pass@8": 0.6477507011049768, + "pass@16": 0.7086113239762887, + "pass@32": 0.7532795279937846, + "pass@64": 0.7862179025168473, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 651414, + "answer_accuracy": 0.7933237529228371, + "avg_response_len": 286.8824020849571, + "resp_tokens_sum": 235564878, + "pass_at_k": { + "pass@1": 0.7933237529228371, + "pass@2": 0.8389930634401326, + "pass@4": 0.8726874711442432, + "pass@8": 0.8977972002351476, + "pass@16": 0.9167457991803611, + "pass@32": 0.9314659835787653, + "pass@64": 0.9432935185698375, + "pass@128": 0.9522992985190959 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9164350076687117, + "pass@2": 0.9361730713492108, + "pass@4": 0.9499516876916461, + "pass@8": 0.9618509716297367, + "pass@16": 0.9718630754382047, + "pass@32": 0.9796931696551145, + "pass@64": 0.986608430448021, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973958333333334, + "pass@2": 0.9992700131233596, + "pass@4": 0.9999176963674994, + "pass@8": 0.9999987979293221, + "pass@16": 0.9999999998399193, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9998897084517542, + "pass@4": 0.9999981377142946, + "pass@8": 0.9999999995762411, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980570652173913, + "pass@2": 0.9865802949903003, + "pass@4": 0.9889661506985535, + "pass@8": 0.9902841199998196, + "pass@16": 0.9917663843037684, + "pass@32": 0.9936594509266741, + "pass@64": 0.9954604899202021, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.995008058739255, + "pass@2": 0.9963252713038379, + "pass@4": 0.9970167348350797, + "pass@8": 0.9973006814242409, + "pass@16": 0.997492802179556, + "pass@32": 0.9978510028653035, + "pass@64": 0.998567335243553, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9898843930635838, + "pass@2": 0.9931856196804879, + "pass@4": 0.9941058232063475, + "pass@8": 0.9943886704848257, + "pass@16": 0.9945808983691612, + "pass@32": 0.9949421965317783, + "pass@64": 0.9956647398843931, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9714788732394366, + "pass@2": 0.980618207275147, + "pass@4": 0.9870130438800786, + "pass@8": 0.9915782021625016, + "pass@16": 0.9946980159453276, + "pass@32": 0.9967270976372445, + "pass@64": 0.9986017780742946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9722352170658682, + "pass@2": 0.9828459468857563, + "pass@4": 0.9884671896364751, + "pass@8": 0.9924020194851084, + "pass@16": 0.9944314239658919, + "pass@32": 0.9953209435293606, + "pass@64": 0.9962633785477046, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9257341867469879, + "pass@2": 0.9553495725026092, + "pass@4": 0.9682735500494665, + "pass@8": 0.9742548441493746, + "pass@16": 0.978185160269226, + "pass@32": 0.9822869779847758, + "pass@64": 0.9871868036674503, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8492358934169278, + "pass@2": 0.8949850973761508, + "pass@4": 0.9207448059000464, + "pass@8": 0.9354093985604018, + "pass@16": 0.943759425744798, + "pass@32": 0.9496060392426146, + "pass@64": 0.9561026755224603, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.740983893557423, + "pass@2": 0.8052852538653259, + "pass@4": 0.845815909040781, + "pass@8": 0.8738550513505369, + "pass@16": 0.895515341616072, + "pass@32": 0.9155451542423412, + "pass@64": 0.9353134265154689, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7266081871345029, + "pass@2": 0.7990618668554583, + "pass@4": 0.8478731022109073, + "pass@8": 0.8837960270998939, + "pass@16": 0.9105606572428289, + "pass@32": 0.930775272521883, + "pass@64": 0.9463526228698942, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6870501893939394, + "pass@2": 0.7726907361011697, + "pass@4": 0.8339367397257161, + "pass@8": 0.8770493599523325, + "pass@16": 0.9072087108288012, + "pass@32": 0.9277764122574453, + "pass@64": 0.9424641653105227, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5663728632478633, + "pass@2": 0.668794655876349, + "pass@4": 0.74511328258113, + "pass@8": 0.8007343935830623, + "pass@16": 0.8421667071713081, + "pass@32": 0.8733083123762971, + "pass@64": 0.8957480080042212, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5376067546583851, + "pass@2": 0.6423940553626447, + "pass@4": 0.7202086238637875, + "pass@8": 0.7743041381948508, + "pass@16": 0.814345713501841, + "pass@32": 0.8470088033255796, + "pass@64": 0.8726913345902738, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5092830882352941, + "pass@2": 0.6142293162343674, + "pass@4": 0.7005960897351063, + "pass@8": 0.7701591745990866, + "pass@16": 0.8221200586922632, + "pass@32": 0.8587938588657354, + "pass@64": 0.8857339251123276, + "pass@128": 0.9058823529411765 + }, + "19": { + "pass@1": 0.34770275297619047, + "pass@2": 0.44574237790588667, + "pass@4": 0.5338633851014157, + "pass@8": 0.611502029947303, + "pass@16": 0.6783513518981501, + "pass@32": 0.7327937015644227, + "pass@64": 0.7742047877222114, + "pass@128": 0.8035714285714286 + }, + "20": { + "pass@1": 0.35043974550898205, + "pass@2": 0.4594935375312366, + "pass@4": 0.5528386059714089, + "pass@8": 0.6238549348742458, + "pass@16": 0.6762174612820911, + "pass@32": 0.7171414832463683, + "pass@64": 0.7511741428622221, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 634704, + "answer_accuracy": 0.7922391755871545, + "avg_response_len": 307.75241027919793, + "resp_tokens_sum": 246556459, + "pass_at_k": { + "pass@1": 0.7922391755871545, + "pass@2": 0.8386018031672131, + "pass@4": 0.8729426678002086, + "pass@8": 0.8980563358146915, + "pass@16": 0.9163577796796314, + "pass@32": 0.9303137674409322, + "pass@64": 0.9412975982929933, + "pass@128": 0.9496724716408372 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.933535447761194, + "pass@2": 0.9610144405923143, + "pass@4": 0.9764659397052976, + "pass@8": 0.9848856698483921, + "pass@16": 0.9903880371665308, + "pass@32": 0.994788130439683, + "pass@64": 0.9981423348247224, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9970163965473705, + "pass@4": 0.997338775072941, + "pass@8": 0.9976015064053606, + "pass@16": 0.9980534958639686, + "pass@32": 0.9987797625879855, + "pass@64": 0.9996441771309199, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.999924094603306, + "pass@4": 0.9999989030976391, + "pass@8": 0.999999999845404, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9750229779411764, + "pass@2": 0.9829337222093564, + "pass@4": 0.9882893535366903, + "pass@8": 0.9921605412616765, + "pass@16": 0.9948705139434515, + "pass@32": 0.9969973900489489, + "pass@64": 0.998910627773521, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9884660883280757, + "pass@2": 0.9913928407312652, + "pass@4": 0.9938846173408135, + "pass@8": 0.9958811876044368, + "pass@16": 0.9974790678017365, + "pass@32": 0.9989033586934598, + "pass@64": 0.9998111525900316, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9794056566455697, + "pass@2": 0.9909034560948872, + "pass@4": 0.9964178719155359, + "pass@8": 0.9984878997029001, + "pass@16": 0.9994102329414561, + "pass@32": 0.9999166822897716, + "pass@64": 0.9999995619546246, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842342342342343, + "pass@2": 0.9942238005722256, + "pass@4": 0.9981784532001067, + "pass@8": 0.9995914077213195, + "pass@16": 0.9999633430821163, + "pass@32": 0.9999998023813371, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9692021704180064, + "pass@2": 0.983175937412968, + "pass@4": 0.9883279185158123, + "pass@8": 0.9908531152358594, + "pass@16": 0.9925946265968474, + "pass@32": 0.9942363552283263, + "pass@64": 0.9958943717110796, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.915327380952381, + "pass@2": 0.948085786151731, + "pass@4": 0.9642386070788763, + "pass@8": 0.9739282351224063, + "pass@16": 0.9804970648118635, + "pass@32": 0.9846325349921398, + "pass@64": 0.986722881877331, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8619313837920489, + "pass@2": 0.912293969274483, + "pass@4": 0.9437201129675306, + "pass@8": 0.9619229572767756, + "pass@16": 0.9731904957944275, + "pass@32": 0.9818321050690701, + "pass@64": 0.9884511684407877, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7652874228395061, + "pass@2": 0.8321079548216199, + "pass@4": 0.8720276435352986, + "pass@8": 0.896107615787007, + "pass@16": 0.9136146173645132, + "pass@32": 0.929635398081815, + "pass@64": 0.9451059800569461, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7490031528189911, + "pass@2": 0.8191228913058717, + "pass@4": 0.864718364024971, + "pass@8": 0.8938565907497251, + "pass@16": 0.9139148391085333, + "pass@32": 0.9293839114599252, + "pass@64": 0.9403161156625768, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6787471064814815, + "pass@2": 0.7596807560513266, + "pass@4": 0.8202685196757814, + "pass@8": 0.8636306777109823, + "pass@16": 0.8919877814582083, + "pass@32": 0.9093606678534814, + "pass@64": 0.9216089042968235, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.5669478244837758, + "pass@2": 0.6690786646691279, + "pass@4": 0.7455410565382864, + "pass@8": 0.8028333423672143, + "pass@16": 0.8423557007877164, + "pass@32": 0.8684380664265011, + "pass@64": 0.8872958659682277, + "pass@128": 0.9026548672566371 + }, + "17": { + "pass@1": 0.5101891942771084, + "pass@2": 0.6125037798833123, + "pass@4": 0.6895920137241878, + "pass@8": 0.7465083545933107, + "pass@16": 0.7900024434338143, + "pass@32": 0.8252868759565376, + "pass@64": 0.85326005071428, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4936743951612903, + "pass@2": 0.5874368967487934, + "pass@4": 0.6604457161604794, + "pass@8": 0.7140707542442847, + "pass@16": 0.7545067071870171, + "pass@32": 0.7888352910327632, + "pass@64": 0.8177670834201991, + "pass@128": 0.8419354838709677 + }, + "19": { + "pass@1": 0.3340728851963746, + "pass@2": 0.4338164890453644, + "pass@4": 0.5283597530860003, + "pass@8": 0.6090080248672055, + "pass@16": 0.6734085647143847, + "pass@32": 0.7271711226701715, + "pass@64": 0.7742411529076022, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.34025431681681684, + "pass@2": 0.45003221725662673, + "pass@4": 0.5509912418042338, + "pass@8": 0.6353882561628813, + "pass@16": 0.6987590181595222, + "pass@32": 0.7426487056681589, + "pass@64": 0.7730709505896831, + "pass@128": 0.7957957957957958 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3cba19b0b14c7a124426e56e9edb0be8982dea90 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ecad35f2fb9b4f11bfcab65b3efa37ec9d2ad82e325202191c412126f98d71 +size 2707181240 diff --git a/results/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv b/results/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..ba3e1516e9556df311f37d5351b1bab898ee068a --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op17-20,0.9801223958333334,0.62273671875,0.7920246710526315,0.0,0.0,0.0,199.76062586805557,372.483859375,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch1-op17-20/summary.json b/results/composition/SDAR-100M-block4-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f0909cf0fafc05ec1cc2020689d627518d5e578a --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op17-20", + "id_acc": 0.9801223958333334, + "ood_acc": 0.62273671875, + "total_acc": 0.7920246710526315, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.76062586805557, + "ood_avg_resp_len": 372.483859375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 640086, + "answer_accuracy": 0.790495079829276, + "avg_response_len": 277.60216270154916, + "resp_tokens_sum": 224782244, + "pass_at_k": { + "pass@1": 0.790495079829276, + "pass@2": 0.8362411306855028, + "pass@4": 0.8696372919991009, + "pass@8": 0.8945669610675093, + "pass@16": 0.913432249792536, + "pass@32": 0.9280573139004922, + "pass@64": 0.9398972482792994, + "pass@128": 0.9494151122352197 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9192477876106194, + "pass@2": 0.9396735564304464, + "pass@4": 0.9543033865788902, + "pass@8": 0.96485112476411, + "pass@16": 0.9724661046772268, + "pass@32": 0.9786862175801799, + "pass@64": 0.9841720997110378, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9998296339563862, + "pass@2": 0.9999961672431132, + "pass@4": 0.9999999985399022, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.999965009391028, + "pass@4": 0.9999997130519236, + "pass@8": 0.9999999999935767, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.996728515625, + "pass@4": 0.9973700640935508, + "pass@8": 0.9978945203621713, + "pass@16": 0.9986215319076661, + "pass@32": 0.99946584925741, + "pass@64": 0.999956800941041, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9784474206349206, + "pass@2": 0.9849374296962881, + "pass@4": 0.9890754283690729, + "pass@8": 0.9921342884438079, + "pass@16": 0.9943983611486136, + "pass@32": 0.9960648708043484, + "pass@64": 0.9967769255677263, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9922342814371258, + "pass@2": 0.9939438298599651, + "pass@4": 0.9941038252516338, + "pass@8": 0.9941991013702666, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9887435281065089, + "pass@2": 0.9931025922517821, + "pass@4": 0.995356032973689, + "pass@8": 0.99687459097789, + "pass@16": 0.997698902492233, + "pass@32": 0.9983400676934957, + "pass@64": 0.9992661790057259, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9668469551282052, + "pass@2": 0.9809139505097918, + "pass@4": 0.9884821623859518, + "pass@8": 0.9934550534186496, + "pass@16": 0.9968600221493318, + "pass@32": 0.9988651869448563, + "pass@64": 0.9998085724581219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9736575704225352, + "pass@2": 0.987104566374626, + "pass@4": 0.9918345429532572, + "pass@8": 0.9940607002019898, + "pass@16": 0.9952133611440159, + "pass@32": 0.9960034370145185, + "pass@64": 0.9968393035368946, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9049442280453258, + "pass@2": 0.9343667049586221, + "pass@4": 0.9518926340829211, + "pass@8": 0.963721513224062, + "pass@16": 0.9715522567392245, + "pass@32": 0.9772850487323939, + "pass@64": 0.9837385106596693, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8367319915254238, + "pass@2": 0.8906847780150366, + "pass@4": 0.9210060177859121, + "pass@8": 0.9395740180895393, + "pass@16": 0.9513328654341852, + "pass@32": 0.9589354102848824, + "pass@64": 0.9654301877762941, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7604574843260188, + "pass@2": 0.82680597894503, + "pass@4": 0.8719512272956471, + "pass@8": 0.9045499773982028, + "pass@16": 0.9266465799635933, + "pass@32": 0.9410437423571842, + "pass@64": 0.9509158308973618, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.7687889408099688, + "pass@2": 0.8359775523094659, + "pass@4": 0.8736571856672118, + "pass@8": 0.8984409718591754, + "pass@16": 0.9181805371311775, + "pass@32": 0.9350351705266242, + "pass@64": 0.9502913100353946, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.6999638728323699, + "pass@2": 0.7756271761640345, + "pass@4": 0.8292882124380401, + "pass@8": 0.869390789970966, + "pass@16": 0.8979501614129438, + "pass@32": 0.9183570437604774, + "pass@64": 0.9343777336335435, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5773941532258065, + "pass@2": 0.6756885795021594, + "pass@4": 0.7475061202027162, + "pass@8": 0.7991479896927378, + "pass@16": 0.8357307006684896, + "pass@32": 0.8628873839423076, + "pass@64": 0.8835885830672884, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5140895953757225, + "pass@2": 0.6143314624960177, + "pass@4": 0.6916131304576808, + "pass@8": 0.7469650535201684, + "pass@16": 0.7860427028084981, + "pass@32": 0.8170511372956204, + "pass@64": 0.8421452842507103, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.4582589285714286, + "pass@2": 0.5605532902137236, + "pass@4": 0.6421960956666121, + "pass@8": 0.7057777001868978, + "pass@16": 0.7555140844320166, + "pass@32": 0.7930716900033024, + "pass@64": 0.822862301855235, + "pass@128": 0.8514285714285714 + }, + "19": { + "pass@1": 0.3429288663663664, + "pass@2": 0.44343987195857243, + "pass@4": 0.531017780998096, + "pass@8": 0.602912347397304, + "pass@16": 0.6667933588207599, + "pass@32": 0.725113626356638, + "pass@64": 0.775691031987435, + "pass@128": 0.8168168168168168 + }, + "20": { + "pass@1": 0.36878284534534533, + "pass@2": 0.4766399026979738, + "pass@4": 0.570342090403114, + "pass@8": 0.6477507011049768, + "pass@16": 0.7086113239762887, + "pass@32": 0.7532795279937846, + "pass@64": 0.7862179025168473, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 651414, + "answer_accuracy": 0.7933237529228371, + "avg_response_len": 286.8824020849571, + "resp_tokens_sum": 235564878, + "pass_at_k": { + "pass@1": 0.7933237529228371, + "pass@2": 0.8389930634401326, + "pass@4": 0.8726874711442432, + "pass@8": 0.8977972002351476, + "pass@16": 0.9167457991803611, + "pass@32": 0.9314659835787653, + "pass@64": 0.9432935185698375, + "pass@128": 0.9522992985190959 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9164350076687117, + "pass@2": 0.9361730713492108, + "pass@4": 0.9499516876916461, + "pass@8": 0.9618509716297367, + "pass@16": 0.9718630754382047, + "pass@32": 0.9796931696551145, + "pass@64": 0.986608430448021, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973958333333334, + "pass@2": 0.9992700131233596, + "pass@4": 0.9999176963674994, + "pass@8": 0.9999987979293221, + "pass@16": 0.9999999998399193, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9998897084517542, + "pass@4": 0.9999981377142946, + "pass@8": 0.9999999995762411, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980570652173913, + "pass@2": 0.9865802949903003, + "pass@4": 0.9889661506985535, + "pass@8": 0.9902841199998196, + "pass@16": 0.9917663843037684, + "pass@32": 0.9936594509266741, + "pass@64": 0.9954604899202021, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.995008058739255, + "pass@2": 0.9963252713038379, + "pass@4": 0.9970167348350797, + "pass@8": 0.9973006814242409, + "pass@16": 0.997492802179556, + "pass@32": 0.9978510028653035, + "pass@64": 0.998567335243553, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9898843930635838, + "pass@2": 0.9931856196804879, + "pass@4": 0.9941058232063475, + "pass@8": 0.9943886704848257, + "pass@16": 0.9945808983691612, + "pass@32": 0.9949421965317783, + "pass@64": 0.9956647398843931, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9714788732394366, + "pass@2": 0.980618207275147, + "pass@4": 0.9870130438800786, + "pass@8": 0.9915782021625016, + "pass@16": 0.9946980159453276, + "pass@32": 0.9967270976372445, + "pass@64": 0.9986017780742946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9722352170658682, + "pass@2": 0.9828459468857563, + "pass@4": 0.9884671896364751, + "pass@8": 0.9924020194851084, + "pass@16": 0.9944314239658919, + "pass@32": 0.9953209435293606, + "pass@64": 0.9962633785477046, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9257341867469879, + "pass@2": 0.9553495725026092, + "pass@4": 0.9682735500494665, + "pass@8": 0.9742548441493746, + "pass@16": 0.978185160269226, + "pass@32": 0.9822869779847758, + "pass@64": 0.9871868036674503, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8492358934169278, + "pass@2": 0.8949850973761508, + "pass@4": 0.9207448059000464, + "pass@8": 0.9354093985604018, + "pass@16": 0.943759425744798, + "pass@32": 0.9496060392426146, + "pass@64": 0.9561026755224603, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.740983893557423, + "pass@2": 0.8052852538653259, + "pass@4": 0.845815909040781, + "pass@8": 0.8738550513505369, + "pass@16": 0.895515341616072, + "pass@32": 0.9155451542423412, + "pass@64": 0.9353134265154689, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7266081871345029, + "pass@2": 0.7990618668554583, + "pass@4": 0.8478731022109073, + "pass@8": 0.8837960270998939, + "pass@16": 0.9105606572428289, + "pass@32": 0.930775272521883, + "pass@64": 0.9463526228698942, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6870501893939394, + "pass@2": 0.7726907361011697, + "pass@4": 0.8339367397257161, + "pass@8": 0.8770493599523325, + "pass@16": 0.9072087108288012, + "pass@32": 0.9277764122574453, + "pass@64": 0.9424641653105227, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5663728632478633, + "pass@2": 0.668794655876349, + "pass@4": 0.74511328258113, + "pass@8": 0.8007343935830623, + "pass@16": 0.8421667071713081, + "pass@32": 0.8733083123762971, + "pass@64": 0.8957480080042212, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5376067546583851, + "pass@2": 0.6423940553626447, + "pass@4": 0.7202086238637875, + "pass@8": 0.7743041381948508, + "pass@16": 0.814345713501841, + "pass@32": 0.8470088033255796, + "pass@64": 0.8726913345902738, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5092830882352941, + "pass@2": 0.6142293162343674, + "pass@4": 0.7005960897351063, + "pass@8": 0.7701591745990866, + "pass@16": 0.8221200586922632, + "pass@32": 0.8587938588657354, + "pass@64": 0.8857339251123276, + "pass@128": 0.9058823529411765 + }, + "19": { + "pass@1": 0.34770275297619047, + "pass@2": 0.44574237790588667, + "pass@4": 0.5338633851014157, + "pass@8": 0.611502029947303, + "pass@16": 0.6783513518981501, + "pass@32": 0.7327937015644227, + "pass@64": 0.7742047877222114, + "pass@128": 0.8035714285714286 + }, + "20": { + "pass@1": 0.35043974550898205, + "pass@2": 0.4594935375312366, + "pass@4": 0.5528386059714089, + "pass@8": 0.6238549348742458, + "pass@16": 0.6762174612820911, + "pass@32": 0.7171414832463683, + "pass@64": 0.7511741428622221, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 634704, + "answer_accuracy": 0.7922391755871545, + "avg_response_len": 307.75241027919793, + "resp_tokens_sum": 246556459, + "pass_at_k": { + "pass@1": 0.7922391755871545, + "pass@2": 0.8386018031672131, + "pass@4": 0.8729426678002086, + "pass@8": 0.8980563358146915, + "pass@16": 0.9163577796796314, + "pass@32": 0.9303137674409322, + "pass@64": 0.9412975982929933, + "pass@128": 0.9496724716408372 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.933535447761194, + "pass@2": 0.9610144405923143, + "pass@4": 0.9764659397052976, + "pass@8": 0.9848856698483921, + "pass@16": 0.9903880371665308, + "pass@32": 0.994788130439683, + "pass@64": 0.9981423348247224, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9970163965473705, + "pass@4": 0.997338775072941, + "pass@8": 0.9976015064053606, + "pass@16": 0.9980534958639686, + "pass@32": 0.9987797625879855, + "pass@64": 0.9996441771309199, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.999924094603306, + "pass@4": 0.9999989030976391, + "pass@8": 0.999999999845404, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9750229779411764, + "pass@2": 0.9829337222093564, + "pass@4": 0.9882893535366903, + "pass@8": 0.9921605412616765, + "pass@16": 0.9948705139434515, + "pass@32": 0.9969973900489489, + "pass@64": 0.998910627773521, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9884660883280757, + "pass@2": 0.9913928407312652, + "pass@4": 0.9938846173408135, + "pass@8": 0.9958811876044368, + "pass@16": 0.9974790678017365, + "pass@32": 0.9989033586934598, + "pass@64": 0.9998111525900316, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9794056566455697, + "pass@2": 0.9909034560948872, + "pass@4": 0.9964178719155359, + "pass@8": 0.9984878997029001, + "pass@16": 0.9994102329414561, + "pass@32": 0.9999166822897716, + "pass@64": 0.9999995619546246, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842342342342343, + "pass@2": 0.9942238005722256, + "pass@4": 0.9981784532001067, + "pass@8": 0.9995914077213195, + "pass@16": 0.9999633430821163, + "pass@32": 0.9999998023813371, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9692021704180064, + "pass@2": 0.983175937412968, + "pass@4": 0.9883279185158123, + "pass@8": 0.9908531152358594, + "pass@16": 0.9925946265968474, + "pass@32": 0.9942363552283263, + "pass@64": 0.9958943717110796, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.915327380952381, + "pass@2": 0.948085786151731, + "pass@4": 0.9642386070788763, + "pass@8": 0.9739282351224063, + "pass@16": 0.9804970648118635, + "pass@32": 0.9846325349921398, + "pass@64": 0.986722881877331, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8619313837920489, + "pass@2": 0.912293969274483, + "pass@4": 0.9437201129675306, + "pass@8": 0.9619229572767756, + "pass@16": 0.9731904957944275, + "pass@32": 0.9818321050690701, + "pass@64": 0.9884511684407877, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7652874228395061, + "pass@2": 0.8321079548216199, + "pass@4": 0.8720276435352986, + "pass@8": 0.896107615787007, + "pass@16": 0.9136146173645132, + "pass@32": 0.929635398081815, + "pass@64": 0.9451059800569461, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7490031528189911, + "pass@2": 0.8191228913058717, + "pass@4": 0.864718364024971, + "pass@8": 0.8938565907497251, + "pass@16": 0.9139148391085333, + "pass@32": 0.9293839114599252, + "pass@64": 0.9403161156625768, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6787471064814815, + "pass@2": 0.7596807560513266, + "pass@4": 0.8202685196757814, + "pass@8": 0.8636306777109823, + "pass@16": 0.8919877814582083, + "pass@32": 0.9093606678534814, + "pass@64": 0.9216089042968235, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.5669478244837758, + "pass@2": 0.6690786646691279, + "pass@4": 0.7455410565382864, + "pass@8": 0.8028333423672143, + "pass@16": 0.8423557007877164, + "pass@32": 0.8684380664265011, + "pass@64": 0.8872958659682277, + "pass@128": 0.9026548672566371 + }, + "17": { + "pass@1": 0.5101891942771084, + "pass@2": 0.6125037798833123, + "pass@4": 0.6895920137241878, + "pass@8": 0.7465083545933107, + "pass@16": 0.7900024434338143, + "pass@32": 0.8252868759565376, + "pass@64": 0.85326005071428, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4936743951612903, + "pass@2": 0.5874368967487934, + "pass@4": 0.6604457161604794, + "pass@8": 0.7140707542442847, + "pass@16": 0.7545067071870171, + "pass@32": 0.7888352910327632, + "pass@64": 0.8177670834201991, + "pass@128": 0.8419354838709677 + }, + "19": { + "pass@1": 0.3340728851963746, + "pass@2": 0.4338164890453644, + "pass@4": 0.5283597530860003, + "pass@8": 0.6090080248672055, + "pass@16": 0.6734085647143847, + "pass@32": 0.7271711226701715, + "pass@64": 0.7742411529076022, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.34025431681681684, + "pass@2": 0.45003221725662673, + "pass@4": 0.5509912418042338, + "pass@8": 0.6353882561628813, + "pass@16": 0.6987590181595222, + "pass@32": 0.7426487056681589, + "pass@64": 0.7730709505896831, + "pass@128": 0.7957957957957958 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8770eb5524b86ee1967fb7ea68e7dcc8f44632b9 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71ab1bc631ed59050633bae19d19e0d66ae3456d050416a21be5dce0fe4b8a27 +size 1789675303 diff --git a/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json b/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..380b9b3e4e659b236aa5e7da521a47e22300abb2 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9879244791666667, + "avg_response_len": 200.573390625, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9879244791666667, + "pass@2": 0.9915930938320194, + "pass@4": 0.9936487634462353, + "pass@8": 0.9949288086694491, + "pass@16": 0.9958718418730274, + "pass@32": 0.9967797861614692, + "pass@64": 0.9977356694007103, + "pass@128": 0.9984444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9435703125, + "pass@2": 0.9607268700787404, + "pass@4": 0.9712526024559434, + "pass@8": 0.9776772575007111, + "pass@16": 0.9820311480328475, + "pass@32": 0.9859655999840329, + "pass@64": 0.9900584070418624, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996328125, + "pass@2": 0.9999351624015749, + "pass@4": 0.9999966226096737, + "pass@8": 0.9999999926430151, + "pass@16": 0.9999999999999936, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998328125, + "pass@2": 0.9986950049212598, + "pass@4": 0.9989937330333709, + "pass@8": 0.9992152625181484, + "pass@16": 0.9994176619420255, + "pass@32": 0.9996886051712321, + "pass@64": 0.9999404409448819, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875625, + "pass@2": 0.9903880413385826, + "pass@4": 0.9921799958755155, + "pass@8": 0.9936264938768348, + "pass@16": 0.9948819632043239, + "pass@32": 0.996308463535151, + "pass@64": 0.997997987322932, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.995953125, + "pass@2": 0.9969644438976379, + "pass@4": 0.9975323900449944, + "pass@8": 0.997917797864405, + "pass@16": 0.9982963251514316, + "pass@32": 0.998686418855384, + "pass@64": 0.9992519681067729, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.992796875, + "pass@2": 0.9954954478346455, + "pass@4": 0.9970782049118859, + "pass@8": 0.9980214999423029, + "pass@16": 0.998521787995179, + "pass@32": 0.9988284479182614, + "pass@64": 0.9989861762904964, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.987875, + "pass@2": 0.9913147145669291, + "pass@4": 0.9929255646794151, + "pass@8": 0.9939211132530895, + "pass@16": 0.9947769123750306, + "pass@32": 0.9956578151347028, + "pass@64": 0.9964109543118082, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.985609375, + "pass@2": 0.9908181594488191, + "pass@4": 0.9928797574053245, + "pass@8": 0.9939798604265343, + "pass@16": 0.9949207781564106, + "pass@32": 0.9958827248544599, + "pass@64": 0.9969750905876388, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9435703125, + "2": 0.9999921875, + "3": 0.9996328125, + "4": 0.998328125, + "5": 0.9875625, + "6": 0.995953125, + "7": 0.992796875, + "8": 0.987875, + "9": 0.985609375 + }, + "per_op_avg_response_len": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 374119, + "answer_accuracy": 0.9871005361364404, + "avg_response_len": 191.1615717874029, + "resp_tokens_sum": 72451765, + "pass_at_k": { + "pass@1": 0.9871005361364404, + "pass@2": 0.9910629106866963, + "pass@4": 0.9933770916046847, + "pass@8": 0.9948866569837124, + "pass@16": 0.9960565960798013, + "pass@32": 0.9972200967202242, + "pass@64": 0.9984194158450799, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9426622418879056, + "pass@2": 0.9608814281234755, + "pass@4": 0.9727997515244226, + "pass@8": 0.9800427227199637, + "pass@16": 0.9848555713566579, + "pass@32": 0.9893375479172565, + "pass@64": 0.9948648057621975, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995068359375, + "pass@2": 0.995943036417323, + "pass@4": 0.996855512396888, + "pass@8": 0.9975476953695743, + "pass@16": 0.9981801935688303, + "pass@32": 0.9990268911601001, + "pass@64": 0.9998138779527558, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9858382936507937, + "pass@2": 0.9894134326959131, + "pass@4": 0.9919654816957404, + "pass@8": 0.993967779218521, + "pass@16": 0.9953518900581235, + "pass@32": 0.9962822752680421, + "pass@64": 0.9967815120663723, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9967230892545617, + "pass@4": 0.9970756080639619, + "pass@8": 0.9971929848408553, + "pass@16": 0.9973802395197485, + "pass@32": 0.9977544910179641, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9944526627218935, + "pass@2": 0.9979470484088896, + "pass@4": 0.9992837274778522, + "pass@8": 0.9998536405673738, + "pass@16": 0.9999944055596715, + "pass@32": 0.9999999969008433, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9830729166666666, + "pass@2": 0.9861033559206539, + "pass@4": 0.987899911849961, + "pass@8": 0.9900541498080317, + "pass@16": 0.9924767953718153, + "pass@32": 0.9948968439236284, + "pass@64": 0.9965094689481032, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9885343309859155, + "pass@2": 0.9931147415992014, + "pass@4": 0.9947651726632762, + "pass@8": 0.9954927131433333, + "pass@16": 0.9964173683303595, + "pass@32": 0.9977688493163707, + "pass@64": 0.9992201397360542, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385895, + "answer_accuracy": 0.9878128071756226, + "avg_response_len": 198.62987641300785, + "resp_tokens_sum": 77595953, + "pass_at_k": { + "pass@1": 0.9878128071756226, + "pass@2": 0.9907689694249806, + "pass@4": 0.9925758746899758, + "pass@8": 0.9937724075737618, + "pass@16": 0.994610426375753, + "pass@32": 0.9954936548420763, + "pass@64": 0.9965385954078559, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.929879217791411, + "pass@2": 0.9455011986135936, + "pass@4": 0.9572367023907287, + "pass@8": 0.9653763793871797, + "pass@16": 0.9709467034438971, + "pass@32": 0.9763425037418643, + "pass@64": 0.9817604399544487, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997395833333333, + "pass@2": 0.999989188141255, + "pass@4": 0.9999999801161218, + "pass@8": 0.9999999999999979, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999799800820015, + "pass@4": 0.9999999084803749, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941123188405797, + "pass@2": 0.9961107782722813, + "pass@4": 0.9970168766947609, + "pass@8": 0.997577219367406, + "pass@16": 0.9980644436436175, + "pass@32": 0.998786836422768, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970898997134671, + "pass@2": 0.9972238595086073, + "pass@4": 0.9973116384270019, + "pass@8": 0.9974829659544705, + "pass@16": 0.9978086997721273, + "pass@32": 0.9983924824583174, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956195809248555, + "pass@2": 0.996662859564881, + "pass@4": 0.9970522314587844, + "pass@8": 0.9971089149255284, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9900308098591549, + "pass@2": 0.9928277836309195, + "pass@4": 0.9939063929508812, + "pass@8": 0.9944474182396722, + "pass@16": 0.9947159353923489, + "pass@32": 0.9950704220578349, + "pass@64": 0.995774647887324, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9818488023952096, + "pass@2": 0.9868720973407518, + "pass@4": 0.9893204078344998, + "pass@8": 0.9909175880114821, + "pass@16": 0.9920082924256072, + "pass@32": 0.993079471807874, + "pass@64": 0.994766372753843, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378075, + "answer_accuracy": 0.9888553523602277, + "avg_response_len": 211.88909231670573, + "resp_tokens_sum": 81012828, + "pass_at_k": { + "pass@1": 0.9888553523602277, + "pass@2": 0.9929607202075135, + "pass@4": 0.9950143064013551, + "pass@8": 0.9961521589491618, + "pass@16": 0.9969775609528468, + "pass@32": 0.9976574270125961, + "pass@64": 0.9982809980262249, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9578125, + "pass@2": 0.9753870901398526, + "pass@4": 0.9833263334620483, + "pass@8": 0.98725395468186, + "pass@16": 0.9899596537917327, + "pass@32": 0.9919179552842842, + "pass@64": 0.9932696551144083, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999225583090379, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9825137867647059, + "pass@2": 0.9854840927512737, + "pass@4": 0.9874707255343083, + "pass@8": 0.9893014786507219, + "pass@16": 0.991217308173432, + "pass@32": 0.9938179066466555, + "pass@64": 0.997452524312115, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995070977917981, + "pass@2": 0.9969331391738493, + "pass@4": 0.998256703913588, + "pass@8": 0.9991602076007862, + "pass@16": 0.9997983877960986, + "pass@32": 0.9999919321054616, + "pass@64": 0.9999999987471165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9879351265822784, + "pass@2": 0.9915949242499751, + "pass@4": 0.9947475725937738, + "pass@8": 0.9970610280579045, + "pass@16": 0.9984926549110091, + "pass@32": 0.9994571169803051, + "pass@64": 0.9999562540838489, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9900760135135135, + "pass@2": 0.9945843973895155, + "pass@4": 0.996588656710704, + "pass@8": 0.9969831382519523, + "pass@16": 0.9969969824466679, + "pass@32": 0.9969969969969951, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.9924345519912906, + "pass@4": 0.9945501765052679, + "pass@8": 0.9955417133916913, + "pass@16": 0.9963403303182642, + "pass@32": 0.9967403207823743, + "pass@64": 0.9967844774326561, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.43462734375, + "avg_response_len": 332.18273828125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.43462734375, + "pass@2": 0.5005182947834617, + "pass@4": 0.5548893307274047, + "pass@8": 0.6003465607833282, + "pass@16": 0.6390786153650679, + "pass@32": 0.6726684092686726, + "pass@64": 0.7024698115574631, + "pass@128": 0.73 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9058125, + "pass@2": 0.9348855807086626, + "pass@4": 0.9523219686914133, + "pass@8": 0.9639877513162247, + "pass@16": 0.9719139636312137, + "pass@32": 0.9772367936401185, + "pass@64": 0.9810516493173799, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.83046875, + "pass@2": 0.8840001230314961, + "pass@4": 0.9133514972815886, + "pass@8": 0.9323962777434657, + "pass@16": 0.9465618674560089, + "pass@32": 0.9567864902434278, + "pass@64": 0.9644186160238337, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.6527734375, + "pass@2": 0.7388490403543305, + "pass@4": 0.7958859667229103, + "pass@8": 0.8354775748363091, + "pass@16": 0.8643046264976756, + "pass@32": 0.8860494112925553, + "pass@64": 0.9030548966211872, + "pass@128": 0.917 + }, + "14": { + "pass@1": 0.5455625, + "pass@2": 0.6375738188976379, + "pass@4": 0.7070053682039741, + "pass@8": 0.7590985628570953, + "pass@16": 0.7975958195184475, + "pass@32": 0.8266908787586564, + "pass@64": 0.8494597774974879, + "pass@128": 0.868 + }, + "15": { + "pass@1": 0.406609375, + "pass@2": 0.4940346948818898, + "pass@4": 0.5684210074990619, + "pass@8": 0.6314350453515915, + "pass@16": 0.6841069770667974, + "pass@32": 0.7270360115703599, + "pass@64": 0.7626391528642439, + "pass@128": 0.794 + }, + "16": { + "pass@1": 0.2661484375, + "pass@2": 0.33698781988188975, + "pass@4": 0.4029694417885262, + "pass@8": 0.46251322880057405, + "pass@16": 0.5149965944758812, + "pass@32": 0.5617009345680936, + "pass@64": 0.6059889851985129, + "pass@128": 0.652 + }, + "17": { + "pass@1": 0.2140859375, + "pass@2": 0.28265403543307094, + "pass@4": 0.3472788755155604, + "pass@8": 0.40412559729262215, + "pass@16": 0.4549240276430866, + "pass@32": 0.501694346448931, + "pass@64": 0.545903991761549, + "pass@128": 0.589 + }, + "18": { + "pass@1": 0.2059765625, + "pass@2": 0.2638793061023622, + "pass@4": 0.31758961445444284, + "pass@8": 0.3670262508999904, + "pass@16": 0.41459444172398247, + "pass@32": 0.4628889802277007, + "pass@64": 0.5113301409316344, + "pass@128": 0.557 + }, + "19": { + "pass@1": 0.1695625, + "pass@2": 0.22803469488188977, + "pass@4": 0.2843005737720284, + "pass@8": 0.33478337444162465, + "pass@16": 0.37877477451174985, + "pass@32": 0.4178315933259912, + "pass@64": 0.45472330372441766, + "pass@128": 0.492 + }, + "20": { + "pass@1": 0.1492734375, + "pass@2": 0.20428383366141709, + "pass@4": 0.2597689933445819, + "pass@8": 0.3126219442937622, + "pass@16": 0.3630130611257066, + "pass@32": 0.40876865261089623, + "pass@64": 0.44612760163432663, + "pass@128": 0.476 + } + }, + "per_op_accuracy": { + "11": 0.9058125, + "12": 0.83046875, + "13": 0.6527734375, + "14": 0.5455625, + "15": 0.406609375, + "16": 0.2661484375, + "17": 0.2140859375, + "18": 0.2059765625, + "19": 0.1695625, + "20": 0.1492734375 + }, + "per_op_avg_response_len": { + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 185959, + "answer_accuracy": 0.43173987741456166, + "avg_response_len": 315.00408618127784, + "resp_tokens_sum": 135678560, + "pass_at_k": { + "pass@1": 0.43173987741456166, + "pass@2": 0.4975053673175705, + "pass@4": 0.5517866410550846, + "pass@8": 0.5971134428338623, + "pass@16": 0.6360000082863407, + "pass@32": 0.670023386734411, + "pass@64": 0.7002047454766951, + "pass@128": 0.7277860326894502 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9032179532577904, + "pass@2": 0.9327289570832682, + "pass@4": 0.9506815622905493, + "pass@8": 0.9626199544828037, + "pass@16": 0.9704751886621362, + "pass@32": 0.9749374452489946, + "pass@64": 0.9778428066504992, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8037826624293786, + "pass@2": 0.8664708589127629, + "pass@4": 0.9026233156236823, + "pass@8": 0.9260914408190967, + "pass@16": 0.9431713204069097, + "pass@32": 0.9549023146846267, + "pass@64": 0.9638589731267034, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.6582092476489029, + "pass@2": 0.743703024337867, + "pass@4": 0.8042070038071254, + "pass@8": 0.8481824192918073, + "pass@16": 0.877888284808596, + "pass@32": 0.8965888649622713, + "pass@64": 0.9091856827840178, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5494548286604362, + "pass@2": 0.6482951130816593, + "pass@4": 0.7280674673259301, + "pass@8": 0.7889771206999603, + "pass@16": 0.8312085331718211, + "pass@32": 0.8598877871539242, + "pass@64": 0.8815913721972909, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.4004019147398844, + "pass@2": 0.4798765275818123, + "pass@4": 0.5449855960020893, + "pass@8": 0.6041668693159691, + "pass@16": 0.6603312308303183, + "pass@32": 0.711706774366159, + "pass@64": 0.7567645044134567, + "pass@128": 0.7947976878612717 + }, + "16": { + "pass@1": 0.2763860887096774, + "pass@2": 0.35277455867411744, + "pass@4": 0.42198129618878266, + "pass@8": 0.48262699673255544, + "pass@16": 0.5365621998798281, + "pass@32": 0.5864838038611693, + "pass@64": 0.6325214538236035, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.19533507947976878, + "pass@2": 0.25727059248554907, + "pass@4": 0.31503235406629077, + "pass@8": 0.36664014844214665, + "pass@16": 0.415407852939479, + "pass@32": 0.46282528956899566, + "pass@64": 0.5089321507525993, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.18772321428571428, + "pass@2": 0.24447272215972995, + "pass@4": 0.29619767395147045, + "pass@8": 0.34292779478769564, + "pass@16": 0.38793229002272034, + "pass@32": 0.43505393984941876, + "pass@64": 0.48128101171399873, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1686373873873874, + "pass@2": 0.22988933815705478, + "pass@4": 0.28967174852017374, + "pass@8": 0.3425929783847028, + "pass@16": 0.3887203432902438, + "pass@32": 0.42960762813584363, + "pass@64": 0.46859317448539534, + "pass@128": 0.5105105105105106 + }, + "20": { + "pass@1": 0.14841403903903905, + "pass@2": 0.19893675663852825, + "pass@4": 0.24889135141891053, + "pass@8": 0.29487053016690784, + "pass@16": 0.3395049732401222, + "pass@32": 0.38101573199885735, + "pass@64": 0.4152574877923962, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 184995, + "answer_accuracy": 0.42975719223907227, + "avg_response_len": 328.94082431980377, + "resp_tokens_sum": 141597183, + "pass_at_k": { + "pass@1": 0.42975719223907227, + "pass@2": 0.49514535935294074, + "pass@4": 0.5506315189359147, + "pass@8": 0.5981688833153738, + "pass@16": 0.6393251536784574, + "pass@32": 0.675103391235929, + "pass@64": 0.706971814163072, + "pass@128": 0.735950044603033 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.913074171686747, + "pass@2": 0.9404327447585614, + "pass@4": 0.9566515369690232, + "pass@8": 0.9679729746481276, + "pass@16": 0.976421032163366, + "pass@32": 0.9826638897843468, + "pass@64": 0.9870178875177636, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8431622257053292, + "pass@2": 0.8853485301014486, + "pass@4": 0.9075407620481607, + "pass@8": 0.9225588387029565, + "pass@16": 0.9338780820659167, + "pass@32": 0.9427151597704617, + "pass@64": 0.9499720050474115, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6195509453781513, + "pass@2": 0.7090015632237144, + "pass@4": 0.7696727071405988, + "pass@8": 0.8128208903813218, + "pass@16": 0.8456239738232122, + "pass@32": 0.8723095721456559, + "pass@64": 0.8960096112676782, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5342881944444444, + "pass@2": 0.6228624176912091, + "pass@4": 0.68985191626266, + "pass@8": 0.7400756479989421, + "pass@16": 0.777373163396424, + "pass@32": 0.8060593902488095, + "pass@64": 0.8296373557844222, + "pass@128": 0.847953216374269 + }, + "15": { + "pass@1": 0.40404829545454546, + "pass@2": 0.4952815557146265, + "pass@4": 0.5744040432445942, + "pass@8": 0.6418894193523699, + "pass@16": 0.6989534805926838, + "pass@32": 0.7435311343356013, + "pass@64": 0.7784359269113569, + "pass@128": 0.8090909090909091 + }, + "16": { + "pass@1": 0.2513577279202279, + "pass@2": 0.321735073580546, + "pass@4": 0.3875065542020923, + "pass@8": 0.4469587390581188, + "pass@16": 0.5009118793512886, + "pass@32": 0.5501861383741561, + "pass@64": 0.5973992977953705, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.21858016304347827, + "pass@2": 0.28791013045923586, + "pass@4": 0.3574839279600918, + "pass@8": 0.42173292646814153, + "pass@16": 0.4792274741006837, + "pass@32": 0.5322220984044989, + "pass@64": 0.5842848428673304, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.20774356617647058, + "pass@2": 0.2646114375868455, + "pass@4": 0.3206001789298397, + "pass@8": 0.3763666437779109, + "pass@16": 0.4341574245513161, + "pass@32": 0.49379397470457037, + "pass@64": 0.5518441577798592, + "pass@128": 0.6058823529411764 + }, + "19": { + "pass@1": 0.1776181175595238, + "pass@2": 0.23588433281777288, + "pass@4": 0.291822026431964, + "pass@8": 0.34046070760558633, + "pass@16": 0.3797265269874325, + "pass@32": 0.4125415160967138, + "pass@64": 0.44444983941437916, + "pass@128": 0.4791666666666667 + }, + "20": { + "pass@1": 0.1407185628742515, + "pass@2": 0.19786669279551142, + "pass@4": 0.25815344816553615, + "pass@8": 0.31686300365274095, + "pass@16": 0.3719599344412786, + "pass@32": 0.41902785399935966, + "pass@64": 0.4536853513921381, + "pass@128": 0.47604790419161674 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 185369, + "answer_accuracy": 0.44260247937041564, + "avg_response_len": 353.18173613233495, + "resp_tokens_sum": 147918162, + "pass_at_k": { + "pass@1": 0.44260247937041564, + "pass@2": 0.509139224727106, + "pass@4": 0.562456437023918, + "pass@8": 0.605909816047592, + "pass@16": 0.6419913306682059, + "pass@32": 0.6728859081903985, + "pass@64": 0.700172046459371, + "pass@128": 0.726161369193154 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9010664682539683, + "pass@2": 0.9314558336457947, + "pass@4": 0.9495970378702663, + "pass@8": 0.9613202533352903, + "pass@16": 0.9687759979531508, + "pass@32": 0.9740935366946681, + "pass@64": 0.9783593648058921, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8469753440366973, + "pass@2": 0.9016613390642684, + "pass@4": 0.9306340686771954, + "pass@8": 0.9488184652821481, + "pass@16": 0.9626058466450654, + "pass@32": 0.97255331767062, + "pass@64": 0.979117645036257, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.6840277777777778, + "pass@2": 0.7669574614805097, + "pass@4": 0.8165764693186498, + "pass@8": 0.8479331025188009, + "pass@16": 0.8715139042865675, + "pass@32": 0.8908118707209612, + "pass@64": 0.9047815820695196, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5532965504451038, + "pass@2": 0.642291218836889, + "pass@4": 0.7043512042448701, + "pass@8": 0.7499437255097021, + "pass@16": 0.7861016572365503, + "pass@32": 0.8160076783891812, + "pass@64": 0.8389702413765118, + "pass@128": 0.857566765578635 + }, + "15": { + "pass@1": 0.41584683641975306, + "pass@2": 0.5078842683727031, + "pass@4": 0.5873538796655046, + "pass@8": 0.6499067906851369, + "pass@16": 0.6943756870491424, + "pass@32": 0.7266055349040754, + "pass@64": 0.7528233902976535, + "pass@128": 0.7777777777777778 + }, + "16": { + "pass@1": 0.2721008480825959, + "pass@2": 0.33834423559333837, + "pass@4": 0.40159421665212186, + "pass@8": 0.4602251988321013, + "pass@16": 0.5098591234832823, + "pass@32": 0.5509605333386498, + "pass@64": 0.5906200029115647, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.22926863704819278, + "pass@2": 0.3040100856180628, + "pass@4": 0.3709875186911878, + "pass@8": 0.42611476990632063, + "pass@16": 0.47253513212532294, + "pass@32": 0.5125942185897905, + "pass@64": 0.5472100849333396, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.22464717741935483, + "pass@2": 0.28498698247396503, + "pass@4": 0.33843989592026796, + "pass@8": 0.38398988335421697, + "pass@16": 0.4232406963502674, + "pass@32": 0.46041983832532407, + "pass@64": 0.5008218489889754, + "pass@128": 0.5451612903225806 + }, + "19": { + "pass@1": 0.1623158987915408, + "pass@2": 0.2182006327758879, + "pass@4": 0.2712618750261958, + "pass@8": 0.32116348907565456, + "pass@16": 0.36780298226078906, + "pass@32": 0.4113543919886992, + "pass@64": 0.45119827969048176, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.15871340090090091, + "pass@2": 0.21606732224350333, + "pass@4": 0.27226703193632334, + "pass@8": 0.32611956314764734, + "pass@16": 0.3775474082082862, + "pass@32": 0.42623156342189406, + "pass@64": 0.46941726977322745, + "pass@128": 0.5075075075075075 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6967154605263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6967154605263158, + "pass@2": 0.7331326732801338, + "pass@4": 0.7627227462257813, + "pass@8": 0.7872539413609735, + "pass@16": 0.8080859331846313, + "pass@32": 0.826194850954748, + "pass@64": 0.8423325863253173, + "pass@128": 0.8571578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9435703125, + "pass@2": 0.9607268700787404, + "pass@4": 0.9712526024559434, + "pass@8": 0.9776772575007111, + "pass@16": 0.9820311480328475, + "pass@32": 0.9859655999840329, + "pass@64": 0.9900584070418624, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996328125, + "pass@2": 0.9999351624015749, + "pass@4": 0.9999966226096737, + "pass@8": 0.9999999926430151, + "pass@16": 0.9999999999999936, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998328125, + "pass@2": 0.9986950049212598, + "pass@4": 0.9989937330333709, + "pass@8": 0.9992152625181484, + "pass@16": 0.9994176619420255, + "pass@32": 0.9996886051712321, + "pass@64": 0.9999404409448819, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875625, + "pass@2": 0.9903880413385826, + "pass@4": 0.9921799958755155, + "pass@8": 0.9936264938768348, + "pass@16": 0.9948819632043239, + "pass@32": 0.996308463535151, + "pass@64": 0.997997987322932, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.995953125, + "pass@2": 0.9969644438976379, + "pass@4": 0.9975323900449944, + "pass@8": 0.997917797864405, + "pass@16": 0.9982963251514316, + "pass@32": 0.998686418855384, + "pass@64": 0.9992519681067729, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.992796875, + "pass@2": 0.9954954478346455, + "pass@4": 0.9970782049118859, + "pass@8": 0.9980214999423029, + "pass@16": 0.998521787995179, + "pass@32": 0.9988284479182614, + "pass@64": 0.9989861762904964, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.987875, + "pass@2": 0.9913147145669291, + "pass@4": 0.9929255646794151, + "pass@8": 0.9939211132530895, + "pass@16": 0.9947769123750306, + "pass@32": 0.9956578151347028, + "pass@64": 0.9964109543118082, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.985609375, + "pass@2": 0.9908181594488191, + "pass@4": 0.9928797574053245, + "pass@8": 0.9939798604265343, + "pass@16": 0.9949207781564106, + "pass@32": 0.9958827248544599, + "pass@64": 0.9969750905876388, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.9058125, + "pass@2": 0.9348855807086626, + "pass@4": 0.9523219686914133, + "pass@8": 0.9639877513162247, + "pass@16": 0.9719139636312137, + "pass@32": 0.9772367936401185, + "pass@64": 0.9810516493173799, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.83046875, + "pass@2": 0.8840001230314961, + "pass@4": 0.9133514972815886, + "pass@8": 0.9323962777434657, + "pass@16": 0.9465618674560089, + "pass@32": 0.9567864902434278, + "pass@64": 0.9644186160238337, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.6527734375, + "pass@2": 0.7388490403543305, + "pass@4": 0.7958859667229103, + "pass@8": 0.8354775748363091, + "pass@16": 0.8643046264976756, + "pass@32": 0.8860494112925553, + "pass@64": 0.9030548966211872, + "pass@128": 0.917 + }, + "14": { + "pass@1": 0.5455625, + "pass@2": 0.6375738188976379, + "pass@4": 0.7070053682039741, + "pass@8": 0.7590985628570953, + "pass@16": 0.7975958195184475, + "pass@32": 0.8266908787586564, + "pass@64": 0.8494597774974879, + "pass@128": 0.868 + }, + "15": { + "pass@1": 0.406609375, + "pass@2": 0.4940346948818898, + "pass@4": 0.5684210074990619, + "pass@8": 0.6314350453515915, + "pass@16": 0.6841069770667974, + "pass@32": 0.7270360115703599, + "pass@64": 0.7626391528642439, + "pass@128": 0.794 + }, + "16": { + "pass@1": 0.2661484375, + "pass@2": 0.33698781988188975, + "pass@4": 0.4029694417885262, + "pass@8": 0.46251322880057405, + "pass@16": 0.5149965944758812, + "pass@32": 0.5617009345680936, + "pass@64": 0.6059889851985129, + "pass@128": 0.652 + }, + "17": { + "pass@1": 0.2140859375, + "pass@2": 0.28265403543307094, + "pass@4": 0.3472788755155604, + "pass@8": 0.40412559729262215, + "pass@16": 0.4549240276430866, + "pass@32": 0.501694346448931, + "pass@64": 0.545903991761549, + "pass@128": 0.589 + }, + "18": { + "pass@1": 0.2059765625, + "pass@2": 0.2638793061023622, + "pass@4": 0.31758961445444284, + "pass@8": 0.3670262508999904, + "pass@16": 0.41459444172398247, + "pass@32": 0.4628889802277007, + "pass@64": 0.5113301409316344, + "pass@128": 0.557 + }, + "19": { + "pass@1": 0.1695625, + "pass@2": 0.22803469488188977, + "pass@4": 0.2843005737720284, + "pass@8": 0.33478337444162465, + "pass@16": 0.37877477451174985, + "pass@32": 0.4178315933259912, + "pass@64": 0.45472330372441766, + "pass@128": 0.492 + }, + "20": { + "pass@1": 0.1492734375, + "pass@2": 0.20428383366141709, + "pass@4": 0.2597689933445819, + "pass@8": 0.3126219442937622, + "pass@16": 0.3630130611257066, + "pass@32": 0.40876865261089623, + "pass@64": 0.44612760163432663, + "pass@128": 0.476 + } + }, + "per_op_accuracy": { + "10": 0.9435703125, + "2": 0.9999921875, + "3": 0.9996328125, + "4": 0.998328125, + "5": 0.9875625, + "6": 0.995953125, + "7": 0.992796875, + "8": 0.987875, + "9": 0.985609375, + "11": 0.9058125, + "12": 0.83046875, + "13": 0.6527734375, + "14": 0.5455625, + "15": 0.406609375, + "16": 0.2661484375, + "17": 0.2140859375, + "18": 0.2059765625, + "19": 0.1695625, + "20": 0.1492734375 + }, + "per_op_avg_response_len": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125, + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125, + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 560078, + "answer_accuracy": 0.691686591052798, + "avg_response_len": 257.03733228936136, + "resp_tokens_sum": 208130325, + "pass_at_k": { + "pass@1": 0.691686591052798, + "pass@2": 0.7285240024607798, + "pass@4": 0.7584811279468562, + "pass@8": 0.7832984708290734, + "pass@16": 0.8045310794934998, + "pass@32": 0.823173791139723, + "pass@64": 0.8397895761692062, + "pass@128": 0.8548846032247865 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9426622418879056, + "pass@2": 0.9608814281234755, + "pass@4": 0.9727997515244226, + "pass@8": 0.9800427227199637, + "pass@16": 0.9848555713566579, + "pass@32": 0.9893375479172565, + "pass@64": 0.9948648057621975, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995068359375, + "pass@2": 0.995943036417323, + "pass@4": 0.996855512396888, + "pass@8": 0.9975476953695743, + "pass@16": 0.9981801935688303, + "pass@32": 0.9990268911601001, + "pass@64": 0.9998138779527558, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9858382936507937, + "pass@2": 0.9894134326959131, + "pass@4": 0.9919654816957404, + "pass@8": 0.993967779218521, + "pass@16": 0.9953518900581235, + "pass@32": 0.9962822752680421, + "pass@64": 0.9967815120663723, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9967230892545617, + "pass@4": 0.9970756080639619, + "pass@8": 0.9971929848408553, + "pass@16": 0.9973802395197485, + "pass@32": 0.9977544910179641, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9944526627218935, + "pass@2": 0.9979470484088896, + "pass@4": 0.9992837274778522, + "pass@8": 0.9998536405673738, + "pass@16": 0.9999944055596715, + "pass@32": 0.9999999969008433, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9830729166666666, + "pass@2": 0.9861033559206539, + "pass@4": 0.987899911849961, + "pass@8": 0.9900541498080317, + "pass@16": 0.9924767953718153, + "pass@32": 0.9948968439236284, + "pass@64": 0.9965094689481032, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9885343309859155, + "pass@2": 0.9931147415992014, + "pass@4": 0.9947651726632762, + "pass@8": 0.9954927131433333, + "pass@16": 0.9964173683303595, + "pass@32": 0.9977688493163707, + "pass@64": 0.9992201397360542, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9032179532577904, + "pass@2": 0.9327289570832682, + "pass@4": 0.9506815622905493, + "pass@8": 0.9626199544828037, + "pass@16": 0.9704751886621362, + "pass@32": 0.9749374452489946, + "pass@64": 0.9778428066504992, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8037826624293786, + "pass@2": 0.8664708589127629, + "pass@4": 0.9026233156236823, + "pass@8": 0.9260914408190967, + "pass@16": 0.9431713204069097, + "pass@32": 0.9549023146846267, + "pass@64": 0.9638589731267034, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.6582092476489029, + "pass@2": 0.743703024337867, + "pass@4": 0.8042070038071254, + "pass@8": 0.8481824192918073, + "pass@16": 0.877888284808596, + "pass@32": 0.8965888649622713, + "pass@64": 0.9091856827840178, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5494548286604362, + "pass@2": 0.6482951130816593, + "pass@4": 0.7280674673259301, + "pass@8": 0.7889771206999603, + "pass@16": 0.8312085331718211, + "pass@32": 0.8598877871539242, + "pass@64": 0.8815913721972909, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.4004019147398844, + "pass@2": 0.4798765275818123, + "pass@4": 0.5449855960020893, + "pass@8": 0.6041668693159691, + "pass@16": 0.6603312308303183, + "pass@32": 0.711706774366159, + "pass@64": 0.7567645044134567, + "pass@128": 0.7947976878612717 + }, + "16": { + "pass@1": 0.2763860887096774, + "pass@2": 0.35277455867411744, + "pass@4": 0.42198129618878266, + "pass@8": 0.48262699673255544, + "pass@16": 0.5365621998798281, + "pass@32": 0.5864838038611693, + "pass@64": 0.6325214538236035, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.19533507947976878, + "pass@2": 0.25727059248554907, + "pass@4": 0.31503235406629077, + "pass@8": 0.36664014844214665, + "pass@16": 0.415407852939479, + "pass@32": 0.46282528956899566, + "pass@64": 0.5089321507525993, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.18772321428571428, + "pass@2": 0.24447272215972995, + "pass@4": 0.29619767395147045, + "pass@8": 0.34292779478769564, + "pass@16": 0.38793229002272034, + "pass@32": 0.43505393984941876, + "pass@64": 0.48128101171399873, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1686373873873874, + "pass@2": 0.22988933815705478, + "pass@4": 0.28967174852017374, + "pass@8": 0.3425929783847028, + "pass@16": 0.3887203432902438, + "pass@32": 0.42960762813584363, + "pass@64": 0.46859317448539534, + "pass@128": 0.5105105105105106 + }, + "20": { + "pass@1": 0.14841403903903905, + "pass@2": 0.19893675663852825, + "pass@4": 0.24889135141891053, + "pass@8": 0.29487053016690784, + "pass@16": 0.3395049732401222, + "pass@32": 0.38101573199885735, + "pass@64": 0.4152574877923962, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 570890, + "answer_accuracy": 0.6952576968043648, + "avg_response_len": 266.9440958690569, + "resp_tokens_sum": 219193136, + "pass_at_k": { + "pass@1": 0.6952576968043648, + "pass@2": 0.7309432171767659, + "pass@4": 0.7608909380725237, + "pass@8": 0.786381191349136, + "pass@16": 0.8083556528635224, + "pass@32": 0.8275322430716204, + "pass@64": 0.8447360879524896, + "pass@128": 0.8603273577552611 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.929879217791411, + "pass@2": 0.9455011986135936, + "pass@4": 0.9572367023907287, + "pass@8": 0.9653763793871797, + "pass@16": 0.9709467034438971, + "pass@32": 0.9763425037418643, + "pass@64": 0.9817604399544487, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997395833333333, + "pass@2": 0.999989188141255, + "pass@4": 0.9999999801161218, + "pass@8": 0.9999999999999979, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999799800820015, + "pass@4": 0.9999999084803749, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941123188405797, + "pass@2": 0.9961107782722813, + "pass@4": 0.9970168766947609, + "pass@8": 0.997577219367406, + "pass@16": 0.9980644436436175, + "pass@32": 0.998786836422768, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970898997134671, + "pass@2": 0.9972238595086073, + "pass@4": 0.9973116384270019, + "pass@8": 0.9974829659544705, + "pass@16": 0.9978086997721273, + "pass@32": 0.9983924824583174, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956195809248555, + "pass@2": 0.996662859564881, + "pass@4": 0.9970522314587844, + "pass@8": 0.9971089149255284, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9900308098591549, + "pass@2": 0.9928277836309195, + "pass@4": 0.9939063929508812, + "pass@8": 0.9944474182396722, + "pass@16": 0.9947159353923489, + "pass@32": 0.9950704220578349, + "pass@64": 0.995774647887324, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9818488023952096, + "pass@2": 0.9868720973407518, + "pass@4": 0.9893204078344998, + "pass@8": 0.9909175880114821, + "pass@16": 0.9920082924256072, + "pass@32": 0.993079471807874, + "pass@64": 0.994766372753843, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.913074171686747, + "pass@2": 0.9404327447585614, + "pass@4": 0.9566515369690232, + "pass@8": 0.9679729746481276, + "pass@16": 0.976421032163366, + "pass@32": 0.9826638897843468, + "pass@64": 0.9870178875177636, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8431622257053292, + "pass@2": 0.8853485301014486, + "pass@4": 0.9075407620481607, + "pass@8": 0.9225588387029565, + "pass@16": 0.9338780820659167, + "pass@32": 0.9427151597704617, + "pass@64": 0.9499720050474115, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6195509453781513, + "pass@2": 0.7090015632237144, + "pass@4": 0.7696727071405988, + "pass@8": 0.8128208903813218, + "pass@16": 0.8456239738232122, + "pass@32": 0.8723095721456559, + "pass@64": 0.8960096112676782, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5342881944444444, + "pass@2": 0.6228624176912091, + "pass@4": 0.68985191626266, + "pass@8": 0.7400756479989421, + "pass@16": 0.777373163396424, + "pass@32": 0.8060593902488095, + "pass@64": 0.8296373557844222, + "pass@128": 0.847953216374269 + }, + "15": { + "pass@1": 0.40404829545454546, + "pass@2": 0.4952815557146265, + "pass@4": 0.5744040432445942, + "pass@8": 0.6418894193523699, + "pass@16": 0.6989534805926838, + "pass@32": 0.7435311343356013, + "pass@64": 0.7784359269113569, + "pass@128": 0.8090909090909091 + }, + "16": { + "pass@1": 0.2513577279202279, + "pass@2": 0.321735073580546, + "pass@4": 0.3875065542020923, + "pass@8": 0.4469587390581188, + "pass@16": 0.5009118793512886, + "pass@32": 0.5501861383741561, + "pass@64": 0.5973992977953705, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.21858016304347827, + "pass@2": 0.28791013045923586, + "pass@4": 0.3574839279600918, + "pass@8": 0.42173292646814153, + "pass@16": 0.4792274741006837, + "pass@32": 0.5322220984044989, + "pass@64": 0.5842848428673304, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.20774356617647058, + "pass@2": 0.2646114375868455, + "pass@4": 0.3206001789298397, + "pass@8": 0.3763666437779109, + "pass@16": 0.4341574245513161, + "pass@32": 0.49379397470457037, + "pass@64": 0.5518441577798592, + "pass@128": 0.6058823529411764 + }, + "19": { + "pass@1": 0.1776181175595238, + "pass@2": 0.23588433281777288, + "pass@4": 0.291822026431964, + "pass@8": 0.34046070760558633, + "pass@16": 0.3797265269874325, + "pass@32": 0.4125415160967138, + "pass@64": 0.44444983941437916, + "pass@128": 0.4791666666666667 + }, + "20": { + "pass@1": 0.1407185628742515, + "pass@2": 0.19786669279551142, + "pass@4": 0.25815344816553615, + "pass@8": 0.31686300365274095, + "pass@16": 0.3719599344412786, + "pass@32": 0.41902785399935966, + "pass@64": 0.4536853513921381, + "pass@128": 0.47604790419161674 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 563444, + "answer_accuracy": 0.7032922591468286, + "avg_response_len": 285.7522542538744, + "resp_tokens_sum": 228930990, + "pass_at_k": { + "pass@1": 0.7032922591468286, + "pass@2": 0.7400347043564306, + "pass@4": 0.768887233609696, + "pass@8": 0.7921462560934432, + "pass@16": 0.8114023979090202, + "pass@32": 0.8278775245383643, + "pass@64": 0.842439411586423, + "pass@128": 0.8562070618309634 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9578125, + "pass@2": 0.9753870901398526, + "pass@4": 0.9833263334620483, + "pass@8": 0.98725395468186, + "pass@16": 0.9899596537917327, + "pass@32": 0.9919179552842842, + "pass@64": 0.9932696551144083, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999225583090379, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9825137867647059, + "pass@2": 0.9854840927512737, + "pass@4": 0.9874707255343083, + "pass@8": 0.9893014786507219, + "pass@16": 0.991217308173432, + "pass@32": 0.9938179066466555, + "pass@64": 0.997452524312115, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995070977917981, + "pass@2": 0.9969331391738493, + "pass@4": 0.998256703913588, + "pass@8": 0.9991602076007862, + "pass@16": 0.9997983877960986, + "pass@32": 0.9999919321054616, + "pass@64": 0.9999999987471165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9879351265822784, + "pass@2": 0.9915949242499751, + "pass@4": 0.9947475725937738, + "pass@8": 0.9970610280579045, + "pass@16": 0.9984926549110091, + "pass@32": 0.9994571169803051, + "pass@64": 0.9999562540838489, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9900760135135135, + "pass@2": 0.9945843973895155, + "pass@4": 0.996588656710704, + "pass@8": 0.9969831382519523, + "pass@16": 0.9969969824466679, + "pass@32": 0.9969969969969951, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.9924345519912906, + "pass@4": 0.9945501765052679, + "pass@8": 0.9955417133916913, + "pass@16": 0.9963403303182642, + "pass@32": 0.9967403207823743, + "pass@64": 0.9967844774326561, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9010664682539683, + "pass@2": 0.9314558336457947, + "pass@4": 0.9495970378702663, + "pass@8": 0.9613202533352903, + "pass@16": 0.9687759979531508, + "pass@32": 0.9740935366946681, + "pass@64": 0.9783593648058921, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8469753440366973, + "pass@2": 0.9016613390642684, + "pass@4": 0.9306340686771954, + "pass@8": 0.9488184652821481, + "pass@16": 0.9626058466450654, + "pass@32": 0.97255331767062, + "pass@64": 0.979117645036257, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.6840277777777778, + "pass@2": 0.7669574614805097, + "pass@4": 0.8165764693186498, + "pass@8": 0.8479331025188009, + "pass@16": 0.8715139042865675, + "pass@32": 0.8908118707209612, + "pass@64": 0.9047815820695196, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5532965504451038, + "pass@2": 0.642291218836889, + "pass@4": 0.7043512042448701, + "pass@8": 0.7499437255097021, + "pass@16": 0.7861016572365503, + "pass@32": 0.8160076783891812, + "pass@64": 0.8389702413765118, + "pass@128": 0.857566765578635 + }, + "15": { + "pass@1": 0.41584683641975306, + "pass@2": 0.5078842683727031, + "pass@4": 0.5873538796655046, + "pass@8": 0.6499067906851369, + "pass@16": 0.6943756870491424, + "pass@32": 0.7266055349040754, + "pass@64": 0.7528233902976535, + "pass@128": 0.7777777777777778 + }, + "16": { + "pass@1": 0.2721008480825959, + "pass@2": 0.33834423559333837, + "pass@4": 0.40159421665212186, + "pass@8": 0.4602251988321013, + "pass@16": 0.5098591234832823, + "pass@32": 0.5509605333386498, + "pass@64": 0.5906200029115647, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.22926863704819278, + "pass@2": 0.3040100856180628, + "pass@4": 0.3709875186911878, + "pass@8": 0.42611476990632063, + "pass@16": 0.47253513212532294, + "pass@32": 0.5125942185897905, + "pass@64": 0.5472100849333396, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.22464717741935483, + "pass@2": 0.28498698247396503, + "pass@4": 0.33843989592026796, + "pass@8": 0.38398988335421697, + "pass@16": 0.4232406963502674, + "pass@32": 0.46041983832532407, + "pass@64": 0.5008218489889754, + "pass@128": 0.5451612903225806 + }, + "19": { + "pass@1": 0.1623158987915408, + "pass@2": 0.2182006327758879, + "pass@4": 0.2712618750261958, + "pass@8": 0.32116348907565456, + "pass@16": 0.36780298226078906, + "pass@32": 0.4113543919886992, + "pass@64": 0.45119827969048176, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.15871340090090091, + "pass@2": 0.21606732224350333, + "pass@4": 0.27226703193632334, + "pass@8": 0.32611956314764734, + "pass@16": 0.3775474082082862, + "pass@32": 0.42623156342189406, + "pass@64": 0.46941726977322745, + "pass@128": 0.5075075075075075 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0a72522e265864a288b7d03500e35747eeee456a --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:219d5b12b92f5e4410e071b60ab0382bde11ebc229a1ce92f7024879e7c1be5b +size 2580150323 diff --git a/results/composition/SDAR-100M-block4-epoch1-op7-10/summary.csv b/results/composition/SDAR-100M-block4-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..404bcd97ae08565e5f51dcb6ac56c9986f56de7b --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op7-10,0.9879244791666667,0.43462734375,0.6967154605263158,0.0,0.0,0.0,200.573390625,332.18273828125,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch1-op7-10/summary.json b/results/composition/SDAR-100M-block4-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ea1971e59e91ee13035f29707b41113fc19db2cc --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op7-10", + "id_acc": 0.9879244791666667, + "ood_acc": 0.43462734375, + "total_acc": 0.6967154605263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.573390625, + "ood_avg_resp_len": 332.18273828125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 560078, + "answer_accuracy": 0.691686591052798, + "avg_response_len": 257.03733228936136, + "resp_tokens_sum": 208130325, + "pass_at_k": { + "pass@1": 0.691686591052798, + "pass@2": 0.7285240024607798, + "pass@4": 0.7584811279468562, + "pass@8": 0.7832984708290734, + "pass@16": 0.8045310794934998, + "pass@32": 0.823173791139723, + "pass@64": 0.8397895761692062, + "pass@128": 0.8548846032247865 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9426622418879056, + "pass@2": 0.9608814281234755, + "pass@4": 0.9727997515244226, + "pass@8": 0.9800427227199637, + "pass@16": 0.9848555713566579, + "pass@32": 0.9893375479172565, + "pass@64": 0.9948648057621975, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995068359375, + "pass@2": 0.995943036417323, + "pass@4": 0.996855512396888, + "pass@8": 0.9975476953695743, + "pass@16": 0.9981801935688303, + "pass@32": 0.9990268911601001, + "pass@64": 0.9998138779527558, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9858382936507937, + "pass@2": 0.9894134326959131, + "pass@4": 0.9919654816957404, + "pass@8": 0.993967779218521, + "pass@16": 0.9953518900581235, + "pass@32": 0.9962822752680421, + "pass@64": 0.9967815120663723, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9967230892545617, + "pass@4": 0.9970756080639619, + "pass@8": 0.9971929848408553, + "pass@16": 0.9973802395197485, + "pass@32": 0.9977544910179641, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9944526627218935, + "pass@2": 0.9979470484088896, + "pass@4": 0.9992837274778522, + "pass@8": 0.9998536405673738, + "pass@16": 0.9999944055596715, + "pass@32": 0.9999999969008433, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9830729166666666, + "pass@2": 0.9861033559206539, + "pass@4": 0.987899911849961, + "pass@8": 0.9900541498080317, + "pass@16": 0.9924767953718153, + "pass@32": 0.9948968439236284, + "pass@64": 0.9965094689481032, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9885343309859155, + "pass@2": 0.9931147415992014, + "pass@4": 0.9947651726632762, + "pass@8": 0.9954927131433333, + "pass@16": 0.9964173683303595, + "pass@32": 0.9977688493163707, + "pass@64": 0.9992201397360542, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9032179532577904, + "pass@2": 0.9327289570832682, + "pass@4": 0.9506815622905493, + "pass@8": 0.9626199544828037, + "pass@16": 0.9704751886621362, + "pass@32": 0.9749374452489946, + "pass@64": 0.9778428066504992, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8037826624293786, + "pass@2": 0.8664708589127629, + "pass@4": 0.9026233156236823, + "pass@8": 0.9260914408190967, + "pass@16": 0.9431713204069097, + "pass@32": 0.9549023146846267, + "pass@64": 0.9638589731267034, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.6582092476489029, + "pass@2": 0.743703024337867, + "pass@4": 0.8042070038071254, + "pass@8": 0.8481824192918073, + "pass@16": 0.877888284808596, + "pass@32": 0.8965888649622713, + "pass@64": 0.9091856827840178, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5494548286604362, + "pass@2": 0.6482951130816593, + "pass@4": 0.7280674673259301, + "pass@8": 0.7889771206999603, + "pass@16": 0.8312085331718211, + "pass@32": 0.8598877871539242, + "pass@64": 0.8815913721972909, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.4004019147398844, + "pass@2": 0.4798765275818123, + "pass@4": 0.5449855960020893, + "pass@8": 0.6041668693159691, + "pass@16": 0.6603312308303183, + "pass@32": 0.711706774366159, + "pass@64": 0.7567645044134567, + "pass@128": 0.7947976878612717 + }, + "16": { + "pass@1": 0.2763860887096774, + "pass@2": 0.35277455867411744, + "pass@4": 0.42198129618878266, + "pass@8": 0.48262699673255544, + "pass@16": 0.5365621998798281, + "pass@32": 0.5864838038611693, + "pass@64": 0.6325214538236035, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.19533507947976878, + "pass@2": 0.25727059248554907, + "pass@4": 0.31503235406629077, + "pass@8": 0.36664014844214665, + "pass@16": 0.415407852939479, + "pass@32": 0.46282528956899566, + "pass@64": 0.5089321507525993, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.18772321428571428, + "pass@2": 0.24447272215972995, + "pass@4": 0.29619767395147045, + "pass@8": 0.34292779478769564, + "pass@16": 0.38793229002272034, + "pass@32": 0.43505393984941876, + "pass@64": 0.48128101171399873, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1686373873873874, + "pass@2": 0.22988933815705478, + "pass@4": 0.28967174852017374, + "pass@8": 0.3425929783847028, + "pass@16": 0.3887203432902438, + "pass@32": 0.42960762813584363, + "pass@64": 0.46859317448539534, + "pass@128": 0.5105105105105106 + }, + "20": { + "pass@1": 0.14841403903903905, + "pass@2": 0.19893675663852825, + "pass@4": 0.24889135141891053, + "pass@8": 0.29487053016690784, + "pass@16": 0.3395049732401222, + "pass@32": 0.38101573199885735, + "pass@64": 0.4152574877923962, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 570890, + "answer_accuracy": 0.6952576968043648, + "avg_response_len": 266.9440958690569, + "resp_tokens_sum": 219193136, + "pass_at_k": { + "pass@1": 0.6952576968043648, + "pass@2": 0.7309432171767659, + "pass@4": 0.7608909380725237, + "pass@8": 0.786381191349136, + "pass@16": 0.8083556528635224, + "pass@32": 0.8275322430716204, + "pass@64": 0.8447360879524896, + "pass@128": 0.8603273577552611 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.929879217791411, + "pass@2": 0.9455011986135936, + "pass@4": 0.9572367023907287, + "pass@8": 0.9653763793871797, + "pass@16": 0.9709467034438971, + "pass@32": 0.9763425037418643, + "pass@64": 0.9817604399544487, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997395833333333, + "pass@2": 0.999989188141255, + "pass@4": 0.9999999801161218, + "pass@8": 0.9999999999999979, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999799800820015, + "pass@4": 0.9999999084803749, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941123188405797, + "pass@2": 0.9961107782722813, + "pass@4": 0.9970168766947609, + "pass@8": 0.997577219367406, + "pass@16": 0.9980644436436175, + "pass@32": 0.998786836422768, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970898997134671, + "pass@2": 0.9972238595086073, + "pass@4": 0.9973116384270019, + "pass@8": 0.9974829659544705, + "pass@16": 0.9978086997721273, + "pass@32": 0.9983924824583174, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956195809248555, + "pass@2": 0.996662859564881, + "pass@4": 0.9970522314587844, + "pass@8": 0.9971089149255284, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9900308098591549, + "pass@2": 0.9928277836309195, + "pass@4": 0.9939063929508812, + "pass@8": 0.9944474182396722, + "pass@16": 0.9947159353923489, + "pass@32": 0.9950704220578349, + "pass@64": 0.995774647887324, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9818488023952096, + "pass@2": 0.9868720973407518, + "pass@4": 0.9893204078344998, + "pass@8": 0.9909175880114821, + "pass@16": 0.9920082924256072, + "pass@32": 0.993079471807874, + "pass@64": 0.994766372753843, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.913074171686747, + "pass@2": 0.9404327447585614, + "pass@4": 0.9566515369690232, + "pass@8": 0.9679729746481276, + "pass@16": 0.976421032163366, + "pass@32": 0.9826638897843468, + "pass@64": 0.9870178875177636, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8431622257053292, + "pass@2": 0.8853485301014486, + "pass@4": 0.9075407620481607, + "pass@8": 0.9225588387029565, + "pass@16": 0.9338780820659167, + "pass@32": 0.9427151597704617, + "pass@64": 0.9499720050474115, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6195509453781513, + "pass@2": 0.7090015632237144, + "pass@4": 0.7696727071405988, + "pass@8": 0.8128208903813218, + "pass@16": 0.8456239738232122, + "pass@32": 0.8723095721456559, + "pass@64": 0.8960096112676782, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5342881944444444, + "pass@2": 0.6228624176912091, + "pass@4": 0.68985191626266, + "pass@8": 0.7400756479989421, + "pass@16": 0.777373163396424, + "pass@32": 0.8060593902488095, + "pass@64": 0.8296373557844222, + "pass@128": 0.847953216374269 + }, + "15": { + "pass@1": 0.40404829545454546, + "pass@2": 0.4952815557146265, + "pass@4": 0.5744040432445942, + "pass@8": 0.6418894193523699, + "pass@16": 0.6989534805926838, + "pass@32": 0.7435311343356013, + "pass@64": 0.7784359269113569, + "pass@128": 0.8090909090909091 + }, + "16": { + "pass@1": 0.2513577279202279, + "pass@2": 0.321735073580546, + "pass@4": 0.3875065542020923, + "pass@8": 0.4469587390581188, + "pass@16": 0.5009118793512886, + "pass@32": 0.5501861383741561, + "pass@64": 0.5973992977953705, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.21858016304347827, + "pass@2": 0.28791013045923586, + "pass@4": 0.3574839279600918, + "pass@8": 0.42173292646814153, + "pass@16": 0.4792274741006837, + "pass@32": 0.5322220984044989, + "pass@64": 0.5842848428673304, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.20774356617647058, + "pass@2": 0.2646114375868455, + "pass@4": 0.3206001789298397, + "pass@8": 0.3763666437779109, + "pass@16": 0.4341574245513161, + "pass@32": 0.49379397470457037, + "pass@64": 0.5518441577798592, + "pass@128": 0.6058823529411764 + }, + "19": { + "pass@1": 0.1776181175595238, + "pass@2": 0.23588433281777288, + "pass@4": 0.291822026431964, + "pass@8": 0.34046070760558633, + "pass@16": 0.3797265269874325, + "pass@32": 0.4125415160967138, + "pass@64": 0.44444983941437916, + "pass@128": 0.4791666666666667 + }, + "20": { + "pass@1": 0.1407185628742515, + "pass@2": 0.19786669279551142, + "pass@4": 0.25815344816553615, + "pass@8": 0.31686300365274095, + "pass@16": 0.3719599344412786, + "pass@32": 0.41902785399935966, + "pass@64": 0.4536853513921381, + "pass@128": 0.47604790419161674 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 563444, + "answer_accuracy": 0.7032922591468286, + "avg_response_len": 285.7522542538744, + "resp_tokens_sum": 228930990, + "pass_at_k": { + "pass@1": 0.7032922591468286, + "pass@2": 0.7400347043564306, + "pass@4": 0.768887233609696, + "pass@8": 0.7921462560934432, + "pass@16": 0.8114023979090202, + "pass@32": 0.8278775245383643, + "pass@64": 0.842439411586423, + "pass@128": 0.8562070618309634 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9578125, + "pass@2": 0.9753870901398526, + "pass@4": 0.9833263334620483, + "pass@8": 0.98725395468186, + "pass@16": 0.9899596537917327, + "pass@32": 0.9919179552842842, + "pass@64": 0.9932696551144083, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999225583090379, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9825137867647059, + "pass@2": 0.9854840927512737, + "pass@4": 0.9874707255343083, + "pass@8": 0.9893014786507219, + "pass@16": 0.991217308173432, + "pass@32": 0.9938179066466555, + "pass@64": 0.997452524312115, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995070977917981, + "pass@2": 0.9969331391738493, + "pass@4": 0.998256703913588, + "pass@8": 0.9991602076007862, + "pass@16": 0.9997983877960986, + "pass@32": 0.9999919321054616, + "pass@64": 0.9999999987471165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9879351265822784, + "pass@2": 0.9915949242499751, + "pass@4": 0.9947475725937738, + "pass@8": 0.9970610280579045, + "pass@16": 0.9984926549110091, + "pass@32": 0.9994571169803051, + "pass@64": 0.9999562540838489, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9900760135135135, + "pass@2": 0.9945843973895155, + "pass@4": 0.996588656710704, + "pass@8": 0.9969831382519523, + "pass@16": 0.9969969824466679, + "pass@32": 0.9969969969969951, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.9924345519912906, + "pass@4": 0.9945501765052679, + "pass@8": 0.9955417133916913, + "pass@16": 0.9963403303182642, + "pass@32": 0.9967403207823743, + "pass@64": 0.9967844774326561, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9010664682539683, + "pass@2": 0.9314558336457947, + "pass@4": 0.9495970378702663, + "pass@8": 0.9613202533352903, + "pass@16": 0.9687759979531508, + "pass@32": 0.9740935366946681, + "pass@64": 0.9783593648058921, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8469753440366973, + "pass@2": 0.9016613390642684, + "pass@4": 0.9306340686771954, + "pass@8": 0.9488184652821481, + "pass@16": 0.9626058466450654, + "pass@32": 0.97255331767062, + "pass@64": 0.979117645036257, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.6840277777777778, + "pass@2": 0.7669574614805097, + "pass@4": 0.8165764693186498, + "pass@8": 0.8479331025188009, + "pass@16": 0.8715139042865675, + "pass@32": 0.8908118707209612, + "pass@64": 0.9047815820695196, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5532965504451038, + "pass@2": 0.642291218836889, + "pass@4": 0.7043512042448701, + "pass@8": 0.7499437255097021, + "pass@16": 0.7861016572365503, + "pass@32": 0.8160076783891812, + "pass@64": 0.8389702413765118, + "pass@128": 0.857566765578635 + }, + "15": { + "pass@1": 0.41584683641975306, + "pass@2": 0.5078842683727031, + "pass@4": 0.5873538796655046, + "pass@8": 0.6499067906851369, + "pass@16": 0.6943756870491424, + "pass@32": 0.7266055349040754, + "pass@64": 0.7528233902976535, + "pass@128": 0.7777777777777778 + }, + "16": { + "pass@1": 0.2721008480825959, + "pass@2": 0.33834423559333837, + "pass@4": 0.40159421665212186, + "pass@8": 0.4602251988321013, + "pass@16": 0.5098591234832823, + "pass@32": 0.5509605333386498, + "pass@64": 0.5906200029115647, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.22926863704819278, + "pass@2": 0.3040100856180628, + "pass@4": 0.3709875186911878, + "pass@8": 0.42611476990632063, + "pass@16": 0.47253513212532294, + "pass@32": 0.5125942185897905, + "pass@64": 0.5472100849333396, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.22464717741935483, + "pass@2": 0.28498698247396503, + "pass@4": 0.33843989592026796, + "pass@8": 0.38398988335421697, + "pass@16": 0.4232406963502674, + "pass@32": 0.46041983832532407, + "pass@64": 0.5008218489889754, + "pass@128": 0.5451612903225806 + }, + "19": { + "pass@1": 0.1623158987915408, + "pass@2": 0.2182006327758879, + "pass@4": 0.2712618750261958, + "pass@8": 0.32116348907565456, + "pass@16": 0.36780298226078906, + "pass@32": 0.4113543919886992, + "pass@64": 0.45119827969048176, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.15871340090090091, + "pass@2": 0.21606732224350333, + "pass@4": 0.27226703193632334, + "pass@8": 0.32611956314764734, + "pass@16": 0.3775474082082862, + "pass@32": 0.42623156342189406, + "pass@64": 0.46941726977322745, + "pass@128": 0.5075075075075075 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bd99c328b5ccf5adf865886669a903c60b2bc61a --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66b7368a633c99a01e963fc46e959e8b978635cf497b867939e42a1e1ce1df58 +size 1790330455 diff --git a/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json b/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dec6e6da6a2ae174bc6925fb7b97e98aebb01c4e --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9861276041666667, + "avg_response_len": 200.2339045138889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9861276041666667, + "pass@2": 0.9900604084645647, + "pass@4": 0.9923328891701035, + "pass@8": 0.9939966568330358, + "pass@16": 0.9953889368549511, + "pass@32": 0.9966042772153859, + "pass@64": 0.9975937986453868, + "pass@128": 0.9982222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9413671875, + "pass@2": 0.9560073818897641, + "pass@4": 0.9661297797150363, + "pass@8": 0.973958866476777, + "pass@16": 0.9801564851612712, + "pass@32": 0.9853814436253552, + "pass@64": 0.9896901293154565, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999359375, + "pass@2": 0.999675812007874, + "pass@4": 0.9998979723472066, + "pass@8": 0.9999905979568364, + "pass@16": 0.9999999435227294, + "pass@32": 0.9999999999996563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974140625, + "pass@2": 0.998044783464567, + "pass@4": 0.998237083239595, + "pass@8": 0.9984480653393176, + "pass@16": 0.9987941264550821, + "pass@32": 0.9992680481403238, + "pass@64": 0.9997381448050702, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9842578125, + "pass@2": 0.9879093257874021, + "pass@4": 0.9901990563367077, + "pass@8": 0.9922341196998229, + "pass@16": 0.9942029297347884, + "pass@32": 0.9957719067641455, + "pass@64": 0.9967520479354316, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.994, + "pass@2": 0.9949153543307088, + "pass@4": 0.9953240142482191, + "pass@8": 0.9956690460023842, + "pass@16": 0.9961843259195156, + "pass@32": 0.9968943403441024, + "pass@64": 0.9976233493374305, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.990328125, + "pass@2": 0.9936660925196853, + "pass@4": 0.9954954812523434, + "pass@8": 0.9966721388643296, + "pass@16": 0.9974260781790963, + "pass@32": 0.9980104268701047, + "pass@64": 0.9986299091903094, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.986390625, + "pass@2": 0.9911269685039372, + "pass@4": 0.993347476752906, + "pass@8": 0.9948517012143449, + "pass@16": 0.9962994679373544, + "pass@32": 0.9976768848650488, + "pass@64": 0.9986739346860647, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.982046875, + "pass@2": 0.9891979576771657, + "pass@4": 0.9923651386389201, + "pass@8": 0.9941453759435105, + "pass@16": 0.9954370747847234, + "pass@32": 0.996435444329734, + "pass@64": 0.9972366725387218, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9413671875, + "2": 0.999984375, + "3": 0.999359375, + "4": 0.9974140625, + "5": 0.9842578125, + "6": 0.994, + "7": 0.990328125, + "8": 0.986390625, + "9": 0.982046875 + }, + "per_op_avg_response_len": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373518, + "answer_accuracy": 0.9855148176291794, + "avg_response_len": 190.75554605707532, + "resp_tokens_sum": 72297878, + "pass_at_k": { + "pass@1": 0.9855148176291794, + "pass@2": 0.9896487340678164, + "pass@4": 0.9921482425828996, + "pass@8": 0.9939548453003074, + "pass@16": 0.9953476381048021, + "pass@32": 0.9965963582818315, + "pass@64": 0.9976863434508711, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390440634218289, + "pass@2": 0.9555319170557223, + "pass@4": 0.9674239525369062, + "pass@8": 0.9756565404139677, + "pass@16": 0.9813557770331947, + "pass@32": 0.986617282494939, + "pass@64": 0.9912821110885033, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9941333200049213, + "pass@4": 0.9944950684875327, + "pass@8": 0.9951502049040684, + "pass@16": 0.9962316451721318, + "pass@32": 0.9977126504385122, + "pass@64": 0.999181702515844, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9885260436195474, + "pass@4": 0.9904835181316618, + "pass@8": 0.9919887051387225, + "pass@16": 0.9930441900209205, + "pass@32": 0.9935671832417211, + "pass@64": 0.9936503542147976, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9926553143712575, + "pass@2": 0.9937994336130889, + "pass@4": 0.9941684698656681, + "pass@8": 0.9943757134349077, + "pass@16": 0.9947162761058925, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9921181582840237, + "pass@2": 0.9955388342729348, + "pass@4": 0.9968490602875822, + "pass@8": 0.9975059751526393, + "pass@16": 0.9980238469616003, + "pass@32": 0.9987617116948309, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9820462740384616, + "pass@2": 0.9870359504340805, + "pass@4": 0.990314350730197, + "pass@8": 0.9933464239381959, + "pass@16": 0.9958232946543495, + "pass@32": 0.9976443884867328, + "pass@64": 0.9991607192427966, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9861575704225352, + "pass@2": 0.9930700343795054, + "pass@4": 0.9960230070712991, + "pass@8": 0.9978186757417113, + "pass@16": 0.9990916543037955, + "pass@32": 0.9998066566382701, + "pass@64": 0.9999959302892975, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385461, + "answer_accuracy": 0.9867018553407602, + "avg_response_len": 198.25831166038662, + "resp_tokens_sum": 77450799, + "pass_at_k": { + "pass@1": 0.9867018553407602, + "pass@2": 0.9899563649368948, + "pass@4": 0.991872276030618, + "pass@8": 0.9934139864923256, + "pass@16": 0.9948125983863382, + "pass@32": 0.9960225731229545, + "pass@64": 0.997029666582489, + "pass@128": 0.9977064220183486 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9318682898773006, + "pass@2": 0.9437727193130767, + "pass@4": 0.9541319070162242, + "pass@8": 0.9643895337030387, + "pass@16": 0.9734856182486636, + "pass@32": 0.9804536494144921, + "pass@64": 0.9859565813477191, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999289772727272, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998543823964497, + "pass@2": 0.9997983448259796, + "pass@4": 0.9999962377114103, + "pass@8": 0.9999999993222367, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906929347826087, + "pass@2": 0.9938455722925938, + "pass@4": 0.99476957146661, + "pass@8": 0.9953830646909647, + "pass@16": 0.9963772680388042, + "pass@32": 0.9978842427305282, + "pass@64": 0.9994736049298187, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967541189111748, + "pass@2": 0.9971184543013785, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9941067557803468, + "pass@2": 0.9960366790314508, + "pass@4": 0.996919753256999, + "pass@8": 0.9971032378641861, + "pass@16": 0.9971098202636923, + "pass@32": 0.9971098265895949, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.988556338028169, + "pass@2": 0.99323742652767, + "pass@4": 0.9950167355841083, + "pass@8": 0.9959384773710429, + "pass@16": 0.997143266424544, + "pass@32": 0.9986983950949794, + "pass@64": 0.9998376400133082, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9778255988023952, + "pass@2": 0.9839705431656373, + "pass@4": 0.9873886173098122, + "pass@8": 0.9896570716478766, + "pass@16": 0.9912446822176693, + "pass@32": 0.9922904474891701, + "pass@64": 0.9932692843858927, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 377040, + "answer_accuracy": 0.9861483093404754, + "avg_response_len": 211.64834334198193, + "resp_tokens_sum": 80920781, + "pass_at_k": { + "pass@1": 0.9861483093404754, + "pass@2": 0.9905748071037493, + "pass@4": 0.9929865650477151, + "pass@8": 0.9946334542311802, + "pass@16": 0.9960187562742351, + "pass@32": 0.9972064897538371, + "pass@64": 0.9980784675060925, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9529617537313433, + "pass@2": 0.9683944940651077, + "pass@4": 0.9764956958738361, + "pass@8": 0.9815531680276173, + "pass@16": 0.9854345229789683, + "pass@32": 0.9889262541804372, + "pass@64": 0.9917123824987969, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982233965014577, + "pass@2": 0.9990552041964142, + "pass@4": 0.9997025432863167, + "pass@8": 0.9999725887954414, + "pass@16": 0.9999998353432346, + "pass@32": 0.9999999999989982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999715804438919, + "pass@4": 0.9999998040266896, + "pass@8": 0.9999999999973679, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9767233455882353, + "pass@2": 0.9813144106067622, + "pass@4": 0.9852977822625111, + "pass@8": 0.9892662360668899, + "pass@16": 0.9930701541317976, + "pass@32": 0.9956711185322674, + "pass@64": 0.9968640960499501, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9923846608832808, + "pass@2": 0.9936656244566434, + "pass@4": 0.9945481545682183, + "pass@8": 0.9954181631392943, + "pass@16": 0.9966848255525158, + "pass@32": 0.9982819052559925, + "pass@64": 0.9995942613043955, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842761075949367, + "pass@2": 0.9890673278181997, + "pass@4": 0.992488178000693, + "pass@8": 0.9953082245624335, + "pass@16": 0.997132974983664, + "pass@32": 0.9981929377128227, + "pass@64": 0.9992150520076071, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881522147147147, + "pass@2": 0.9927101067603032, + "pass@4": 0.9944097843625403, + "pass@8": 0.995103475912635, + "pass@16": 0.9958460673407935, + "pass@32": 0.9966183345299415, + "pass@64": 0.9969772614942578, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9818880627009646, + "pass@2": 0.9903920709167786, + "pass@4": 0.9935343181579793, + "pass@8": 0.9947726177646687, + "pass@16": 0.9957678895375398, + "pass@32": 0.9970388159542295, + "pass@64": 0.9983478337657014, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5314875, + "avg_response_len": 351.76516015625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5314875, + "pass@2": 0.6022682824803063, + "pass@4": 0.6591659706880293, + "pass@8": 0.7050961345515291, + "pass@16": 0.7429625058318009, + "pass@32": 0.7753246502977222, + "pass@64": 0.8030656828422016, + "pass@128": 0.8261 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9324765625, + "pass@2": 0.9533142224409449, + "pass@4": 0.9647201501687288, + "pass@8": 0.9716220664213624, + "pass@16": 0.976113902180434, + "pass@32": 0.9794617764117818, + "pass@64": 0.9824683910904393, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.887390625, + "pass@2": 0.9206886072834655, + "pass@4": 0.9376154036370457, + "pass@8": 0.9486329455856432, + "pass@16": 0.9569428807974291, + "pass@32": 0.9633673292075392, + "pass@64": 0.968846758845277, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.7620703125, + "pass@2": 0.8222394192913397, + "pass@4": 0.8591277995875507, + "pass@8": 0.8854334828073739, + "pass@16": 0.9066821609763444, + "pass@32": 0.925073595662466, + "pass@64": 0.9415350592744647, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.701, + "pass@2": 0.7807849409448818, + "pass@4": 0.832770978908886, + "pass@8": 0.86770836585606, + "pass@16": 0.8925915209483687, + "pass@32": 0.9113136378973459, + "pass@64": 0.9257772846219111, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.603734375, + "pass@2": 0.6913080708661427, + "pass@4": 0.7554190513685792, + "pass@8": 0.8031779786366131, + "pass@16": 0.8383422442454967, + "pass@32": 0.8645934027916754, + "pass@64": 0.8849775249687437, + "pass@128": 0.902 + }, + "16": { + "pass@1": 0.421171875, + "pass@2": 0.5247765748031502, + "pass@4": 0.6098393095238099, + "pass@8": 0.6755907587075335, + "pass@16": 0.7269389528502289, + "pass@32": 0.7697787249307066, + "pass@64": 0.8058135993617338, + "pass@128": 0.833 + }, + "17": { + "pass@1": 0.3317109375, + "pass@2": 0.42685334645669337, + "pass@4": 0.5099146227971499, + "pass@8": 0.5782730292294023, + "pass@16": 0.6352742989388074, + "pass@32": 0.686522191097258, + "pass@64": 0.7313242064501279, + "pass@128": 0.765 + }, + "18": { + "pass@1": 0.3008359375, + "pass@2": 0.38705302657480345, + "pass@4": 0.4653752832770899, + "pass@8": 0.5326123127775788, + "pass@16": 0.5900990793779466, + "pass@32": 0.6406784031888708, + "pass@64": 0.6850894214131005, + "pass@128": 0.725 + }, + "19": { + "pass@1": 0.1969375, + "pass@2": 0.2683048720472441, + "pass@4": 0.33789413226471676, + "pass@8": 0.40227891800683, + "pass@16": 0.4607638000638037, + "pass@32": 0.51352503178184, + "pass@64": 0.5616739114507694, + "pass@128": 0.604 + }, + "20": { + "pass@1": 0.177546875, + "pass@2": 0.2473597440944882, + "pass@4": 0.3189829753468313, + "pass@8": 0.38563148748692394, + "pass@16": 0.4458762179390442, + "pass@32": 0.4989324100077299, + "pass@64": 0.5431506709453723, + "pass@128": 0.58 + } + }, + "per_op_accuracy": { + "11": 0.9324765625, + "12": 0.887390625, + "13": 0.7620703125, + "14": 0.701, + "15": 0.603734375, + "16": 0.421171875, + "17": 0.3317109375, + "18": 0.3008359375, + "19": 0.1969375, + "20": 0.177546875 + }, + "per_op_avg_response_len": { + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 230194, + "answer_accuracy": 0.5344400074294205, + "avg_response_len": 333.67903510401186, + "resp_tokens_sum": 143722234, + "pass_at_k": { + "pass@1": 0.5344400074294205, + "pass@2": 0.6035592116039382, + "pass@4": 0.6583647019234046, + "pass@8": 0.7021859925548064, + "pass@16": 0.7387123981798346, + "pass@32": 0.7706893337971354, + "pass@64": 0.7985455630917165, + "pass@128": 0.8222882615156017 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9276071175637394, + "pass@2": 0.948805304922933, + "pass@4": 0.9625242864471968, + "pass@8": 0.9718624532312413, + "pass@16": 0.9782309268492891, + "pass@32": 0.9824962329796952, + "pass@64": 0.9854704070111471, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8833862994350282, + "pass@2": 0.919130772053917, + "pass@4": 0.9369724676894196, + "pass@8": 0.949119166761525, + "pass@16": 0.9589524280731624, + "pass@32": 0.9663844678408191, + "pass@64": 0.9727848261772267, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.77953565830721, + "pass@2": 0.8406290881939132, + "pass@4": 0.8766417460944336, + "pass@8": 0.9007702961787816, + "pass@16": 0.9203564319065582, + "pass@32": 0.9373645301631106, + "pass@64": 0.9519000597968401, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.726927570093458, + "pass@2": 0.8062917003949269, + "pass@4": 0.8548241513619205, + "pass@8": 0.8863406152747126, + "pass@16": 0.9104425624453125, + "pass@32": 0.9293280986882642, + "pass@64": 0.9439192226652879, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6221324060693642, + "pass@2": 0.7063558213099081, + "pass@4": 0.7662719325907954, + "pass@8": 0.8111031131548001, + "pass@16": 0.8462265090737875, + "pass@32": 0.8747750472498352, + "pass@64": 0.8970795874266666, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.4443296370967742, + "pass@2": 0.547864014478029, + "pass@4": 0.6275922671964388, + "pass@8": 0.6861379453842721, + "pass@16": 0.7316585228509355, + "pass@32": 0.7729760509118466, + "pass@64": 0.812534223224478, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.3257541546242775, + "pass@2": 0.4236906035228256, + "pass@4": 0.5111456820245731, + "pass@8": 0.5823636363388676, + "pass@16": 0.6392486711470267, + "pass@32": 0.6881428453312798, + "pass@64": 0.7286531028102093, + "pass@128": 0.7572254335260116 + }, + "18": { + "pass@1": 0.26602678571428573, + "pass@2": 0.34801216254218215, + "pass@4": 0.4255049338475545, + "pass@8": 0.4947810451410119, + "pass@16": 0.55549192818462, + "pass@32": 0.6101924953252681, + "pass@64": 0.658631555139458, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18520082582582584, + "pass@2": 0.2561013277056583, + "pass@4": 0.32643228098739907, + "pass@8": 0.39033346408424197, + "pass@16": 0.4477674798841049, + "pass@32": 0.4997264097323752, + "pass@64": 0.5464207467355886, + "pass@128": 0.5855855855855856 + }, + "20": { + "pass@1": 0.16732357357357358, + "pass@2": 0.2275554343713792, + "pass@4": 0.28745300840773264, + "pass@8": 0.3421696822977434, + "pass@16": 0.39285128747771964, + "pass@32": 0.4406795710163925, + "pass@64": 0.4845239493330482, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 228448, + "answer_accuracy": 0.5307017543859649, + "avg_response_len": 348.2110025460898, + "resp_tokens_sum": 149892301, + "pass_at_k": { + "pass@1": 0.5307017543859649, + "pass@2": 0.6027607857977414, + "pass@4": 0.6611699793658727, + "pass@8": 0.7089237533553516, + "pass@16": 0.7483928661324409, + "pass@32": 0.781609045904677, + "pass@64": 0.809569957323907, + "pass@128": 0.8322925958965209 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9410062123493976, + "pass@2": 0.958402384142871, + "pass@4": 0.9662167536851564, + "pass@8": 0.9709692525310544, + "pass@16": 0.9742474397487791, + "pass@32": 0.9771765082319408, + "pass@64": 0.9803255550853777, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839880485893417, + "pass@2": 0.9137333232542638, + "pass@4": 0.9281012561517581, + "pass@8": 0.9354493476435397, + "pass@16": 0.9396505982108377, + "pass@32": 0.9434898827285836, + "pass@64": 0.9480917933676873, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7396271008403361, + "pass@2": 0.7994693448245437, + "pass@4": 0.8373948400882663, + "pass@8": 0.8673476898330499, + "pass@16": 0.8936367585473864, + "pass@32": 0.9163371027271825, + "pass@64": 0.9352437459860121, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6859237938596491, + "pass@2": 0.7680046881475341, + "pass@4": 0.8229837292816459, + "pass@8": 0.8614744075315885, + "pass@16": 0.888244901857619, + "pass@32": 0.9069925793938199, + "pass@64": 0.920712589392341, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.592779356060606, + "pass@2": 0.6902823759246002, + "pass@4": 0.7625650600493118, + "pass@8": 0.8162019416966678, + "pass@16": 0.8545200049180915, + "pass@32": 0.8813196398182785, + "pass@64": 0.9009509702228317, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.419315349002849, + "pass@2": 0.5241369568387281, + "pass@4": 0.6076210393572596, + "pass@8": 0.6690281184725626, + "pass@16": 0.7159874640286359, + "pass@32": 0.7552395900040212, + "pass@64": 0.7868517864460535, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3528484083850932, + "pass@2": 0.4481074546388222, + "pass@4": 0.5321736775333203, + "pass@8": 0.6030474903401737, + "pass@16": 0.6627250939917279, + "pass@32": 0.7157933575690838, + "pass@64": 0.7650930743179017, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.32125459558823527, + "pass@2": 0.411982905859194, + "pass@4": 0.49527306485586303, + "pass@8": 0.5687953540170917, + "pass@16": 0.6336849247595163, + "pass@32": 0.6896902244753337, + "pass@64": 0.7332202055305614, + "pass@128": 0.7647058823529411 + }, + "19": { + "pass@1": 0.20452008928571427, + "pass@2": 0.27103216101893507, + "pass@4": 0.3350326398932277, + "pass@8": 0.3956876057484762, + "pass@16": 0.4526802546976231, + "pass@32": 0.5058468809363461, + "pass@64": 0.557456349702467, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.17171126497005987, + "pass@2": 0.2459550632985996, + "pass@4": 0.32637963022212046, + "pass@8": 0.4023198144270265, + "pass@16": 0.46881532030483886, + "pass@32": 0.5238391423437389, + "pass@64": 0.5672146906934317, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 221662, + "answer_accuracy": 0.5292586720660146, + "avg_response_len": 374.0183517344132, + "resp_tokens_sum": 156644870, + "pass_at_k": { + "pass@1": 0.5292586720660146, + "pass@2": 0.6004344606106715, + "pass@4": 0.6579302702630552, + "pass@8": 0.7041549199371585, + "pass@16": 0.7417520261734117, + "pass@32": 0.7736325407617366, + "pass@64": 0.8010291082328783, + "pass@128": 0.8236552567237164 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9289434523809523, + "pass@2": 0.9530043119610053, + "pass@4": 0.9656035391409405, + "pass@8": 0.9720407256838862, + "pass@16": 0.9757086825588593, + "pass@32": 0.9784698585299846, + "pass@64": 0.9813627083401868, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8950449159021406, + "pass@2": 0.9291601952852221, + "pass@4": 0.9475928115063601, + "pass@8": 0.960967641143043, + "pass@16": 0.9716366374014452, + "pass@32": 0.9794922171298824, + "pass@64": 0.9848307287285822, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.7696035879629629, + "pass@2": 0.8292227902935744, + "pass@4": 0.8658306317034441, + "pass@8": 0.89026121313563, + "pass@16": 0.9075930135701733, + "pass@32": 0.9225986569963891, + "pass@64": 0.9382621076613159, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6916033011869436, + "pass@2": 0.7694590557022359, + "pass@4": 0.8216973023958051, + "pass@8": 0.8562871839082323, + "pass@16": 0.8799991156323995, + "pass@32": 0.8985396322425133, + "pass@64": 0.91363652395897, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.5952449845679012, + "pass@2": 0.6762832488820844, + "pass@4": 0.736550903937933, + "pass@8": 0.7814495701393551, + "pass@16": 0.8134452792685088, + "pass@32": 0.836684429948149, + "pass@64": 0.8557844677332779, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.4019174041297935, + "pass@2": 0.504326426149165, + "pass@4": 0.5959018344498971, + "pass@8": 0.6727407848216501, + "pass@16": 0.7339622740188452, + "pass@32": 0.7819087110814815, + "pass@64": 0.8193009236565792, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.31741810993975905, + "pass@2": 0.40953553386775415, + "pass@4": 0.48704305015336896, + "pass@8": 0.5499816842368038, + "pass@16": 0.604508368845186, + "pass@32": 0.6564436911909359, + "pass@64": 0.7013562136970772, + "pass@128": 0.7349397590361446 + }, + "18": { + "pass@1": 0.317741935483871, + "pass@2": 0.40378897320294627, + "pass@4": 0.47759907864339524, + "pass@8": 0.5356404084271387, + "pass@16": 0.581367839016435, + "pass@32": 0.6213430758174604, + "pass@64": 0.6621726039803202, + "pass@128": 0.7096774193548387 + }, + "19": { + "pass@1": 0.20104796072507553, + "pass@2": 0.277813667126579, + "pass@4": 0.3523299567727752, + "pass@8": 0.42098742880752116, + "pass@16": 0.48204430116010694, + "pass@32": 0.5352011641883591, + "pass@64": 0.5813005114434726, + "pass@128": 0.6193353474320241 + }, + "20": { + "pass@1": 0.1936233108108108, + "pass@2": 0.268572952874134, + "pass@4": 0.34309407523383884, + "pass@8": 0.412354850640087, + "pass@16": 0.4758931598412825, + "pass@32": 0.5322037216710581, + "pass@64": 0.5776411084860688, + "pass@128": 0.6126126126126126 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7468433388157895, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7468433388157895, + "pass@2": 0.7859592895254661, + "pass@4": 0.8169818794426844, + "pass@8": 0.8419437503690984, + "pass@16": 0.8625329205269855, + "pass@32": 0.880141315679784, + "pass@64": 0.8952105798016025, + "pass@128": 0.9076315789473685 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9413671875, + "pass@2": 0.9560073818897641, + "pass@4": 0.9661297797150363, + "pass@8": 0.973958866476777, + "pass@16": 0.9801564851612712, + "pass@32": 0.9853814436253552, + "pass@64": 0.9896901293154565, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999359375, + "pass@2": 0.999675812007874, + "pass@4": 0.9998979723472066, + "pass@8": 0.9999905979568364, + "pass@16": 0.9999999435227294, + "pass@32": 0.9999999999996563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974140625, + "pass@2": 0.998044783464567, + "pass@4": 0.998237083239595, + "pass@8": 0.9984480653393176, + "pass@16": 0.9987941264550821, + "pass@32": 0.9992680481403238, + "pass@64": 0.9997381448050702, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9842578125, + "pass@2": 0.9879093257874021, + "pass@4": 0.9901990563367077, + "pass@8": 0.9922341196998229, + "pass@16": 0.9942029297347884, + "pass@32": 0.9957719067641455, + "pass@64": 0.9967520479354316, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.994, + "pass@2": 0.9949153543307088, + "pass@4": 0.9953240142482191, + "pass@8": 0.9956690460023842, + "pass@16": 0.9961843259195156, + "pass@32": 0.9968943403441024, + "pass@64": 0.9976233493374305, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.990328125, + "pass@2": 0.9936660925196853, + "pass@4": 0.9954954812523434, + "pass@8": 0.9966721388643296, + "pass@16": 0.9974260781790963, + "pass@32": 0.9980104268701047, + "pass@64": 0.9986299091903094, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.986390625, + "pass@2": 0.9911269685039372, + "pass@4": 0.993347476752906, + "pass@8": 0.9948517012143449, + "pass@16": 0.9962994679373544, + "pass@32": 0.9976768848650488, + "pass@64": 0.9986739346860647, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.982046875, + "pass@2": 0.9891979576771657, + "pass@4": 0.9923651386389201, + "pass@8": 0.9941453759435105, + "pass@16": 0.9954370747847234, + "pass@32": 0.996435444329734, + "pass@64": 0.9972366725387218, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.9324765625, + "pass@2": 0.9533142224409449, + "pass@4": 0.9647201501687288, + "pass@8": 0.9716220664213624, + "pass@16": 0.976113902180434, + "pass@32": 0.9794617764117818, + "pass@64": 0.9824683910904393, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.887390625, + "pass@2": 0.9206886072834655, + "pass@4": 0.9376154036370457, + "pass@8": 0.9486329455856432, + "pass@16": 0.9569428807974291, + "pass@32": 0.9633673292075392, + "pass@64": 0.968846758845277, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.7620703125, + "pass@2": 0.8222394192913397, + "pass@4": 0.8591277995875507, + "pass@8": 0.8854334828073739, + "pass@16": 0.9066821609763444, + "pass@32": 0.925073595662466, + "pass@64": 0.9415350592744647, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.701, + "pass@2": 0.7807849409448818, + "pass@4": 0.832770978908886, + "pass@8": 0.86770836585606, + "pass@16": 0.8925915209483687, + "pass@32": 0.9113136378973459, + "pass@64": 0.9257772846219111, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.603734375, + "pass@2": 0.6913080708661427, + "pass@4": 0.7554190513685792, + "pass@8": 0.8031779786366131, + "pass@16": 0.8383422442454967, + "pass@32": 0.8645934027916754, + "pass@64": 0.8849775249687437, + "pass@128": 0.902 + }, + "16": { + "pass@1": 0.421171875, + "pass@2": 0.5247765748031502, + "pass@4": 0.6098393095238099, + "pass@8": 0.6755907587075335, + "pass@16": 0.7269389528502289, + "pass@32": 0.7697787249307066, + "pass@64": 0.8058135993617338, + "pass@128": 0.833 + }, + "17": { + "pass@1": 0.3317109375, + "pass@2": 0.42685334645669337, + "pass@4": 0.5099146227971499, + "pass@8": 0.5782730292294023, + "pass@16": 0.6352742989388074, + "pass@32": 0.686522191097258, + "pass@64": 0.7313242064501279, + "pass@128": 0.765 + }, + "18": { + "pass@1": 0.3008359375, + "pass@2": 0.38705302657480345, + "pass@4": 0.4653752832770899, + "pass@8": 0.5326123127775788, + "pass@16": 0.5900990793779466, + "pass@32": 0.6406784031888708, + "pass@64": 0.6850894214131005, + "pass@128": 0.725 + }, + "19": { + "pass@1": 0.1969375, + "pass@2": 0.2683048720472441, + "pass@4": 0.33789413226471676, + "pass@8": 0.40227891800683, + "pass@16": 0.4607638000638037, + "pass@32": 0.51352503178184, + "pass@64": 0.5616739114507694, + "pass@128": 0.604 + }, + "20": { + "pass@1": 0.177546875, + "pass@2": 0.2473597440944882, + "pass@4": 0.3189829753468313, + "pass@8": 0.38563148748692394, + "pass@16": 0.4458762179390442, + "pass@32": 0.4989324100077299, + "pass@64": 0.5431506709453723, + "pass@128": 0.58 + } + }, + "per_op_accuracy": { + "10": 0.9413671875, + "2": 0.999984375, + "3": 0.999359375, + "4": 0.9974140625, + "5": 0.9842578125, + "6": 0.994, + "7": 0.990328125, + "8": 0.986390625, + "9": 0.982046875, + "11": 0.9324765625, + "12": 0.887390625, + "13": 0.7620703125, + "14": 0.701, + "15": 0.603734375, + "16": 0.421171875, + "17": 0.3317109375, + "18": 0.3008359375, + "19": 0.1969375, + "20": 0.177546875 + }, + "per_op_avg_response_len": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875, + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875, + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 603712, + "answer_accuracy": 0.7455738223205818, + "avg_response_len": 266.78108204236486, + "resp_tokens_sum": 216020112, + "pass_at_k": { + "pass@1": 0.7455738223205818, + "pass@2": 0.7842754740154935, + "pass@4": 0.8145981928960141, + "pass@8": 0.8387537404175, + "pass@16": 0.8588352159822145, + "pass@32": 0.8764292483559714, + "pass@64": 0.8917570475437393, + "pass@128": 0.9046791021182422 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390440634218289, + "pass@2": 0.9555319170557223, + "pass@4": 0.9674239525369062, + "pass@8": 0.9756565404139677, + "pass@16": 0.9813557770331947, + "pass@32": 0.986617282494939, + "pass@64": 0.9912821110885033, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9941333200049213, + "pass@4": 0.9944950684875327, + "pass@8": 0.9951502049040684, + "pass@16": 0.9962316451721318, + "pass@32": 0.9977126504385122, + "pass@64": 0.999181702515844, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9885260436195474, + "pass@4": 0.9904835181316618, + "pass@8": 0.9919887051387225, + "pass@16": 0.9930441900209205, + "pass@32": 0.9935671832417211, + "pass@64": 0.9936503542147976, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9926553143712575, + "pass@2": 0.9937994336130889, + "pass@4": 0.9941684698656681, + "pass@8": 0.9943757134349077, + "pass@16": 0.9947162761058925, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9921181582840237, + "pass@2": 0.9955388342729348, + "pass@4": 0.9968490602875822, + "pass@8": 0.9975059751526393, + "pass@16": 0.9980238469616003, + "pass@32": 0.9987617116948309, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9820462740384616, + "pass@2": 0.9870359504340805, + "pass@4": 0.990314350730197, + "pass@8": 0.9933464239381959, + "pass@16": 0.9958232946543495, + "pass@32": 0.9976443884867328, + "pass@64": 0.9991607192427966, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9861575704225352, + "pass@2": 0.9930700343795054, + "pass@4": 0.9960230070712991, + "pass@8": 0.9978186757417113, + "pass@16": 0.9990916543037955, + "pass@32": 0.9998066566382701, + "pass@64": 0.9999959302892975, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9276071175637394, + "pass@2": 0.948805304922933, + "pass@4": 0.9625242864471968, + "pass@8": 0.9718624532312413, + "pass@16": 0.9782309268492891, + "pass@32": 0.9824962329796952, + "pass@64": 0.9854704070111471, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8833862994350282, + "pass@2": 0.919130772053917, + "pass@4": 0.9369724676894196, + "pass@8": 0.949119166761525, + "pass@16": 0.9589524280731624, + "pass@32": 0.9663844678408191, + "pass@64": 0.9727848261772267, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.77953565830721, + "pass@2": 0.8406290881939132, + "pass@4": 0.8766417460944336, + "pass@8": 0.9007702961787816, + "pass@16": 0.9203564319065582, + "pass@32": 0.9373645301631106, + "pass@64": 0.9519000597968401, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.726927570093458, + "pass@2": 0.8062917003949269, + "pass@4": 0.8548241513619205, + "pass@8": 0.8863406152747126, + "pass@16": 0.9104425624453125, + "pass@32": 0.9293280986882642, + "pass@64": 0.9439192226652879, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6221324060693642, + "pass@2": 0.7063558213099081, + "pass@4": 0.7662719325907954, + "pass@8": 0.8111031131548001, + "pass@16": 0.8462265090737875, + "pass@32": 0.8747750472498352, + "pass@64": 0.8970795874266666, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.4443296370967742, + "pass@2": 0.547864014478029, + "pass@4": 0.6275922671964388, + "pass@8": 0.6861379453842721, + "pass@16": 0.7316585228509355, + "pass@32": 0.7729760509118466, + "pass@64": 0.812534223224478, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.3257541546242775, + "pass@2": 0.4236906035228256, + "pass@4": 0.5111456820245731, + "pass@8": 0.5823636363388676, + "pass@16": 0.6392486711470267, + "pass@32": 0.6881428453312798, + "pass@64": 0.7286531028102093, + "pass@128": 0.7572254335260116 + }, + "18": { + "pass@1": 0.26602678571428573, + "pass@2": 0.34801216254218215, + "pass@4": 0.4255049338475545, + "pass@8": 0.4947810451410119, + "pass@16": 0.55549192818462, + "pass@32": 0.6101924953252681, + "pass@64": 0.658631555139458, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18520082582582584, + "pass@2": 0.2561013277056583, + "pass@4": 0.32643228098739907, + "pass@8": 0.39033346408424197, + "pass@16": 0.4477674798841049, + "pass@32": 0.4997264097323752, + "pass@64": 0.5464207467355886, + "pass@128": 0.5855855855855856 + }, + "20": { + "pass@1": 0.16732357357357358, + "pass@2": 0.2275554343713792, + "pass@4": 0.28745300840773264, + "pass@8": 0.3421696822977434, + "pass@16": 0.39285128747771964, + "pass@32": 0.4406795710163925, + "pass@64": 0.4845239493330482, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 613909, + "answer_accuracy": 0.7476483339828527, + "avg_response_len": 276.8695196804365, + "resp_tokens_sum": 227343100, + "pass_at_k": { + "pass@1": 0.7476483339828527, + "pass@2": 0.7869729303858387, + "pass@4": 0.8185048834065208, + "pass@8": 0.8442728089335293, + "pass@16": 0.8656295025843366, + "pass@32": 0.8836184122445357, + "pass@64": 0.8987557769119414, + "pass@128": 0.9109898674980514 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9318682898773006, + "pass@2": 0.9437727193130767, + "pass@4": 0.9541319070162242, + "pass@8": 0.9643895337030387, + "pass@16": 0.9734856182486636, + "pass@32": 0.9804536494144921, + "pass@64": 0.9859565813477191, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999289772727272, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998543823964497, + "pass@2": 0.9997983448259796, + "pass@4": 0.9999962377114103, + "pass@8": 0.9999999993222367, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906929347826087, + "pass@2": 0.9938455722925938, + "pass@4": 0.99476957146661, + "pass@8": 0.9953830646909647, + "pass@16": 0.9963772680388042, + "pass@32": 0.9978842427305282, + "pass@64": 0.9994736049298187, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967541189111748, + "pass@2": 0.9971184543013785, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9941067557803468, + "pass@2": 0.9960366790314508, + "pass@4": 0.996919753256999, + "pass@8": 0.9971032378641861, + "pass@16": 0.9971098202636923, + "pass@32": 0.9971098265895949, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.988556338028169, + "pass@2": 0.99323742652767, + "pass@4": 0.9950167355841083, + "pass@8": 0.9959384773710429, + "pass@16": 0.997143266424544, + "pass@32": 0.9986983950949794, + "pass@64": 0.9998376400133082, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9778255988023952, + "pass@2": 0.9839705431656373, + "pass@4": 0.9873886173098122, + "pass@8": 0.9896570716478766, + "pass@16": 0.9912446822176693, + "pass@32": 0.9922904474891701, + "pass@64": 0.9932692843858927, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9410062123493976, + "pass@2": 0.958402384142871, + "pass@4": 0.9662167536851564, + "pass@8": 0.9709692525310544, + "pass@16": 0.9742474397487791, + "pass@32": 0.9771765082319408, + "pass@64": 0.9803255550853777, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839880485893417, + "pass@2": 0.9137333232542638, + "pass@4": 0.9281012561517581, + "pass@8": 0.9354493476435397, + "pass@16": 0.9396505982108377, + "pass@32": 0.9434898827285836, + "pass@64": 0.9480917933676873, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7396271008403361, + "pass@2": 0.7994693448245437, + "pass@4": 0.8373948400882663, + "pass@8": 0.8673476898330499, + "pass@16": 0.8936367585473864, + "pass@32": 0.9163371027271825, + "pass@64": 0.9352437459860121, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6859237938596491, + "pass@2": 0.7680046881475341, + "pass@4": 0.8229837292816459, + "pass@8": 0.8614744075315885, + "pass@16": 0.888244901857619, + "pass@32": 0.9069925793938199, + "pass@64": 0.920712589392341, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.592779356060606, + "pass@2": 0.6902823759246002, + "pass@4": 0.7625650600493118, + "pass@8": 0.8162019416966678, + "pass@16": 0.8545200049180915, + "pass@32": 0.8813196398182785, + "pass@64": 0.9009509702228317, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.419315349002849, + "pass@2": 0.5241369568387281, + "pass@4": 0.6076210393572596, + "pass@8": 0.6690281184725626, + "pass@16": 0.7159874640286359, + "pass@32": 0.7552395900040212, + "pass@64": 0.7868517864460535, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3528484083850932, + "pass@2": 0.4481074546388222, + "pass@4": 0.5321736775333203, + "pass@8": 0.6030474903401737, + "pass@16": 0.6627250939917279, + "pass@32": 0.7157933575690838, + "pass@64": 0.7650930743179017, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.32125459558823527, + "pass@2": 0.411982905859194, + "pass@4": 0.49527306485586303, + "pass@8": 0.5687953540170917, + "pass@16": 0.6336849247595163, + "pass@32": 0.6896902244753337, + "pass@64": 0.7332202055305614, + "pass@128": 0.7647058823529411 + }, + "19": { + "pass@1": 0.20452008928571427, + "pass@2": 0.27103216101893507, + "pass@4": 0.3350326398932277, + "pass@8": 0.3956876057484762, + "pass@16": 0.4526802546976231, + "pass@32": 0.5058468809363461, + "pass@64": 0.557456349702467, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.17171126497005987, + "pass@2": 0.2459550632985996, + "pass@4": 0.32637963022212046, + "pass@8": 0.4023198144270265, + "pass@16": 0.46881532030483886, + "pass@32": 0.5238391423437389, + "pass@64": 0.5672146906934317, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 598702, + "answer_accuracy": 0.747301386004154, + "avg_response_len": 296.53006046293336, + "resp_tokens_sum": 237565651, + "pass_at_k": { + "pass@1": 0.747301386004154, + "pass@2": 0.7866222246264524, + "pass@4": 0.817830118884518, + "pass@8": 0.8427807997799793, + "pass@16": 0.8630964458588545, + "pass@32": 0.8803293590457137, + "pass@64": 0.8950675226999064, + "pass@128": 0.9071736699153219 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9529617537313433, + "pass@2": 0.9683944940651077, + "pass@4": 0.9764956958738361, + "pass@8": 0.9815531680276173, + "pass@16": 0.9854345229789683, + "pass@32": 0.9889262541804372, + "pass@64": 0.9917123824987969, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982233965014577, + "pass@2": 0.9990552041964142, + "pass@4": 0.9997025432863167, + "pass@8": 0.9999725887954414, + "pass@16": 0.9999998353432346, + "pass@32": 0.9999999999989982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999715804438919, + "pass@4": 0.9999998040266896, + "pass@8": 0.9999999999973679, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9767233455882353, + "pass@2": 0.9813144106067622, + "pass@4": 0.9852977822625111, + "pass@8": 0.9892662360668899, + "pass@16": 0.9930701541317976, + "pass@32": 0.9956711185322674, + "pass@64": 0.9968640960499501, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9923846608832808, + "pass@2": 0.9936656244566434, + "pass@4": 0.9945481545682183, + "pass@8": 0.9954181631392943, + "pass@16": 0.9966848255525158, + "pass@32": 0.9982819052559925, + "pass@64": 0.9995942613043955, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842761075949367, + "pass@2": 0.9890673278181997, + "pass@4": 0.992488178000693, + "pass@8": 0.9953082245624335, + "pass@16": 0.997132974983664, + "pass@32": 0.9981929377128227, + "pass@64": 0.9992150520076071, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881522147147147, + "pass@2": 0.9927101067603032, + "pass@4": 0.9944097843625403, + "pass@8": 0.995103475912635, + "pass@16": 0.9958460673407935, + "pass@32": 0.9966183345299415, + "pass@64": 0.9969772614942578, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9818880627009646, + "pass@2": 0.9903920709167786, + "pass@4": 0.9935343181579793, + "pass@8": 0.9947726177646687, + "pass@16": 0.9957678895375398, + "pass@32": 0.9970388159542295, + "pass@64": 0.9983478337657014, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9289434523809523, + "pass@2": 0.9530043119610053, + "pass@4": 0.9656035391409405, + "pass@8": 0.9720407256838862, + "pass@16": 0.9757086825588593, + "pass@32": 0.9784698585299846, + "pass@64": 0.9813627083401868, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8950449159021406, + "pass@2": 0.9291601952852221, + "pass@4": 0.9475928115063601, + "pass@8": 0.960967641143043, + "pass@16": 0.9716366374014452, + "pass@32": 0.9794922171298824, + "pass@64": 0.9848307287285822, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.7696035879629629, + "pass@2": 0.8292227902935744, + "pass@4": 0.8658306317034441, + "pass@8": 0.89026121313563, + "pass@16": 0.9075930135701733, + "pass@32": 0.9225986569963891, + "pass@64": 0.9382621076613159, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6916033011869436, + "pass@2": 0.7694590557022359, + "pass@4": 0.8216973023958051, + "pass@8": 0.8562871839082323, + "pass@16": 0.8799991156323995, + "pass@32": 0.8985396322425133, + "pass@64": 0.91363652395897, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.5952449845679012, + "pass@2": 0.6762832488820844, + "pass@4": 0.736550903937933, + "pass@8": 0.7814495701393551, + "pass@16": 0.8134452792685088, + "pass@32": 0.836684429948149, + "pass@64": 0.8557844677332779, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.4019174041297935, + "pass@2": 0.504326426149165, + "pass@4": 0.5959018344498971, + "pass@8": 0.6727407848216501, + "pass@16": 0.7339622740188452, + "pass@32": 0.7819087110814815, + "pass@64": 0.8193009236565792, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.31741810993975905, + "pass@2": 0.40953553386775415, + "pass@4": 0.48704305015336896, + "pass@8": 0.5499816842368038, + "pass@16": 0.604508368845186, + "pass@32": 0.6564436911909359, + "pass@64": 0.7013562136970772, + "pass@128": 0.7349397590361446 + }, + "18": { + "pass@1": 0.317741935483871, + "pass@2": 0.40378897320294627, + "pass@4": 0.47759907864339524, + "pass@8": 0.5356404084271387, + "pass@16": 0.581367839016435, + "pass@32": 0.6213430758174604, + "pass@64": 0.6621726039803202, + "pass@128": 0.7096774193548387 + }, + "19": { + "pass@1": 0.20104796072507553, + "pass@2": 0.277813667126579, + "pass@4": 0.3523299567727752, + "pass@8": 0.42098742880752116, + "pass@16": 0.48204430116010694, + "pass@32": 0.5352011641883591, + "pass@64": 0.5813005114434726, + "pass@128": 0.6193353474320241 + }, + "20": { + "pass@1": 0.1936233108108108, + "pass@2": 0.268572952874134, + "pass@4": 0.34309407523383884, + "pass@8": 0.412354850640087, + "pass@16": 0.4758931598412825, + "pass@32": 0.5322037216710581, + "pass@64": 0.5776411084860688, + "pass@128": 0.6126126126126126 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3d9f27335a251504baad4b2184baabaa7d5f39e8 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d7b869d5020046086f356b8d79deea63a586b2a7d5b059980100798a9c3866b +size 2638378973 diff --git a/results/composition/SDAR-100M-block4-epoch1-op9-12/summary.csv b/results/composition/SDAR-100M-block4-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9508b7090177ae78b7d07bbfa23457b24d3fe357 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op9-12,0.9861276041666667,0.5314875,0.7468433388157895,0.0,0.0,0.0,200.2339045138889,351.76516015625,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch1-op9-12/summary.json b/results/composition/SDAR-100M-block4-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0908d4d6a2abc32b6ecee09efdb08d04ee9db6e2 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op9-12", + "id_acc": 0.9861276041666667, + "ood_acc": 0.5314875, + "total_acc": 0.7468433388157895, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.2339045138889, + "ood_avg_resp_len": 351.76516015625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 603712, + "answer_accuracy": 0.7455738223205818, + "avg_response_len": 266.78108204236486, + "resp_tokens_sum": 216020112, + "pass_at_k": { + "pass@1": 0.7455738223205818, + "pass@2": 0.7842754740154935, + "pass@4": 0.8145981928960141, + "pass@8": 0.8387537404175, + "pass@16": 0.8588352159822145, + "pass@32": 0.8764292483559714, + "pass@64": 0.8917570475437393, + "pass@128": 0.9046791021182422 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390440634218289, + "pass@2": 0.9555319170557223, + "pass@4": 0.9674239525369062, + "pass@8": 0.9756565404139677, + "pass@16": 0.9813557770331947, + "pass@32": 0.986617282494939, + "pass@64": 0.9912821110885033, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9941333200049213, + "pass@4": 0.9944950684875327, + "pass@8": 0.9951502049040684, + "pass@16": 0.9962316451721318, + "pass@32": 0.9977126504385122, + "pass@64": 0.999181702515844, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9885260436195474, + "pass@4": 0.9904835181316618, + "pass@8": 0.9919887051387225, + "pass@16": 0.9930441900209205, + "pass@32": 0.9935671832417211, + "pass@64": 0.9936503542147976, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9926553143712575, + "pass@2": 0.9937994336130889, + "pass@4": 0.9941684698656681, + "pass@8": 0.9943757134349077, + "pass@16": 0.9947162761058925, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9921181582840237, + "pass@2": 0.9955388342729348, + "pass@4": 0.9968490602875822, + "pass@8": 0.9975059751526393, + "pass@16": 0.9980238469616003, + "pass@32": 0.9987617116948309, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9820462740384616, + "pass@2": 0.9870359504340805, + "pass@4": 0.990314350730197, + "pass@8": 0.9933464239381959, + "pass@16": 0.9958232946543495, + "pass@32": 0.9976443884867328, + "pass@64": 0.9991607192427966, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9861575704225352, + "pass@2": 0.9930700343795054, + "pass@4": 0.9960230070712991, + "pass@8": 0.9978186757417113, + "pass@16": 0.9990916543037955, + "pass@32": 0.9998066566382701, + "pass@64": 0.9999959302892975, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9276071175637394, + "pass@2": 0.948805304922933, + "pass@4": 0.9625242864471968, + "pass@8": 0.9718624532312413, + "pass@16": 0.9782309268492891, + "pass@32": 0.9824962329796952, + "pass@64": 0.9854704070111471, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8833862994350282, + "pass@2": 0.919130772053917, + "pass@4": 0.9369724676894196, + "pass@8": 0.949119166761525, + "pass@16": 0.9589524280731624, + "pass@32": 0.9663844678408191, + "pass@64": 0.9727848261772267, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.77953565830721, + "pass@2": 0.8406290881939132, + "pass@4": 0.8766417460944336, + "pass@8": 0.9007702961787816, + "pass@16": 0.9203564319065582, + "pass@32": 0.9373645301631106, + "pass@64": 0.9519000597968401, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.726927570093458, + "pass@2": 0.8062917003949269, + "pass@4": 0.8548241513619205, + "pass@8": 0.8863406152747126, + "pass@16": 0.9104425624453125, + "pass@32": 0.9293280986882642, + "pass@64": 0.9439192226652879, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6221324060693642, + "pass@2": 0.7063558213099081, + "pass@4": 0.7662719325907954, + "pass@8": 0.8111031131548001, + "pass@16": 0.8462265090737875, + "pass@32": 0.8747750472498352, + "pass@64": 0.8970795874266666, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.4443296370967742, + "pass@2": 0.547864014478029, + "pass@4": 0.6275922671964388, + "pass@8": 0.6861379453842721, + "pass@16": 0.7316585228509355, + "pass@32": 0.7729760509118466, + "pass@64": 0.812534223224478, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.3257541546242775, + "pass@2": 0.4236906035228256, + "pass@4": 0.5111456820245731, + "pass@8": 0.5823636363388676, + "pass@16": 0.6392486711470267, + "pass@32": 0.6881428453312798, + "pass@64": 0.7286531028102093, + "pass@128": 0.7572254335260116 + }, + "18": { + "pass@1": 0.26602678571428573, + "pass@2": 0.34801216254218215, + "pass@4": 0.4255049338475545, + "pass@8": 0.4947810451410119, + "pass@16": 0.55549192818462, + "pass@32": 0.6101924953252681, + "pass@64": 0.658631555139458, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18520082582582584, + "pass@2": 0.2561013277056583, + "pass@4": 0.32643228098739907, + "pass@8": 0.39033346408424197, + "pass@16": 0.4477674798841049, + "pass@32": 0.4997264097323752, + "pass@64": 0.5464207467355886, + "pass@128": 0.5855855855855856 + }, + "20": { + "pass@1": 0.16732357357357358, + "pass@2": 0.2275554343713792, + "pass@4": 0.28745300840773264, + "pass@8": 0.3421696822977434, + "pass@16": 0.39285128747771964, + "pass@32": 0.4406795710163925, + "pass@64": 0.4845239493330482, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 613909, + "answer_accuracy": 0.7476483339828527, + "avg_response_len": 276.8695196804365, + "resp_tokens_sum": 227343100, + "pass_at_k": { + "pass@1": 0.7476483339828527, + "pass@2": 0.7869729303858387, + "pass@4": 0.8185048834065208, + "pass@8": 0.8442728089335293, + "pass@16": 0.8656295025843366, + "pass@32": 0.8836184122445357, + "pass@64": 0.8987557769119414, + "pass@128": 0.9109898674980514 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9318682898773006, + "pass@2": 0.9437727193130767, + "pass@4": 0.9541319070162242, + "pass@8": 0.9643895337030387, + "pass@16": 0.9734856182486636, + "pass@32": 0.9804536494144921, + "pass@64": 0.9859565813477191, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999289772727272, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998543823964497, + "pass@2": 0.9997983448259796, + "pass@4": 0.9999962377114103, + "pass@8": 0.9999999993222367, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906929347826087, + "pass@2": 0.9938455722925938, + "pass@4": 0.99476957146661, + "pass@8": 0.9953830646909647, + "pass@16": 0.9963772680388042, + "pass@32": 0.9978842427305282, + "pass@64": 0.9994736049298187, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967541189111748, + "pass@2": 0.9971184543013785, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9941067557803468, + "pass@2": 0.9960366790314508, + "pass@4": 0.996919753256999, + "pass@8": 0.9971032378641861, + "pass@16": 0.9971098202636923, + "pass@32": 0.9971098265895949, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.988556338028169, + "pass@2": 0.99323742652767, + "pass@4": 0.9950167355841083, + "pass@8": 0.9959384773710429, + "pass@16": 0.997143266424544, + "pass@32": 0.9986983950949794, + "pass@64": 0.9998376400133082, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9778255988023952, + "pass@2": 0.9839705431656373, + "pass@4": 0.9873886173098122, + "pass@8": 0.9896570716478766, + "pass@16": 0.9912446822176693, + "pass@32": 0.9922904474891701, + "pass@64": 0.9932692843858927, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9410062123493976, + "pass@2": 0.958402384142871, + "pass@4": 0.9662167536851564, + "pass@8": 0.9709692525310544, + "pass@16": 0.9742474397487791, + "pass@32": 0.9771765082319408, + "pass@64": 0.9803255550853777, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839880485893417, + "pass@2": 0.9137333232542638, + "pass@4": 0.9281012561517581, + "pass@8": 0.9354493476435397, + "pass@16": 0.9396505982108377, + "pass@32": 0.9434898827285836, + "pass@64": 0.9480917933676873, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7396271008403361, + "pass@2": 0.7994693448245437, + "pass@4": 0.8373948400882663, + "pass@8": 0.8673476898330499, + "pass@16": 0.8936367585473864, + "pass@32": 0.9163371027271825, + "pass@64": 0.9352437459860121, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6859237938596491, + "pass@2": 0.7680046881475341, + "pass@4": 0.8229837292816459, + "pass@8": 0.8614744075315885, + "pass@16": 0.888244901857619, + "pass@32": 0.9069925793938199, + "pass@64": 0.920712589392341, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.592779356060606, + "pass@2": 0.6902823759246002, + "pass@4": 0.7625650600493118, + "pass@8": 0.8162019416966678, + "pass@16": 0.8545200049180915, + "pass@32": 0.8813196398182785, + "pass@64": 0.9009509702228317, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.419315349002849, + "pass@2": 0.5241369568387281, + "pass@4": 0.6076210393572596, + "pass@8": 0.6690281184725626, + "pass@16": 0.7159874640286359, + "pass@32": 0.7552395900040212, + "pass@64": 0.7868517864460535, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3528484083850932, + "pass@2": 0.4481074546388222, + "pass@4": 0.5321736775333203, + "pass@8": 0.6030474903401737, + "pass@16": 0.6627250939917279, + "pass@32": 0.7157933575690838, + "pass@64": 0.7650930743179017, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.32125459558823527, + "pass@2": 0.411982905859194, + "pass@4": 0.49527306485586303, + "pass@8": 0.5687953540170917, + "pass@16": 0.6336849247595163, + "pass@32": 0.6896902244753337, + "pass@64": 0.7332202055305614, + "pass@128": 0.7647058823529411 + }, + "19": { + "pass@1": 0.20452008928571427, + "pass@2": 0.27103216101893507, + "pass@4": 0.3350326398932277, + "pass@8": 0.3956876057484762, + "pass@16": 0.4526802546976231, + "pass@32": 0.5058468809363461, + "pass@64": 0.557456349702467, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.17171126497005987, + "pass@2": 0.2459550632985996, + "pass@4": 0.32637963022212046, + "pass@8": 0.4023198144270265, + "pass@16": 0.46881532030483886, + "pass@32": 0.5238391423437389, + "pass@64": 0.5672146906934317, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 598702, + "answer_accuracy": 0.747301386004154, + "avg_response_len": 296.53006046293336, + "resp_tokens_sum": 237565651, + "pass_at_k": { + "pass@1": 0.747301386004154, + "pass@2": 0.7866222246264524, + "pass@4": 0.817830118884518, + "pass@8": 0.8427807997799793, + "pass@16": 0.8630964458588545, + "pass@32": 0.8803293590457137, + "pass@64": 0.8950675226999064, + "pass@128": 0.9071736699153219 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9529617537313433, + "pass@2": 0.9683944940651077, + "pass@4": 0.9764956958738361, + "pass@8": 0.9815531680276173, + "pass@16": 0.9854345229789683, + "pass@32": 0.9889262541804372, + "pass@64": 0.9917123824987969, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982233965014577, + "pass@2": 0.9990552041964142, + "pass@4": 0.9997025432863167, + "pass@8": 0.9999725887954414, + "pass@16": 0.9999998353432346, + "pass@32": 0.9999999999989982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999715804438919, + "pass@4": 0.9999998040266896, + "pass@8": 0.9999999999973679, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9767233455882353, + "pass@2": 0.9813144106067622, + "pass@4": 0.9852977822625111, + "pass@8": 0.9892662360668899, + "pass@16": 0.9930701541317976, + "pass@32": 0.9956711185322674, + "pass@64": 0.9968640960499501, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9923846608832808, + "pass@2": 0.9936656244566434, + "pass@4": 0.9945481545682183, + "pass@8": 0.9954181631392943, + "pass@16": 0.9966848255525158, + "pass@32": 0.9982819052559925, + "pass@64": 0.9995942613043955, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842761075949367, + "pass@2": 0.9890673278181997, + "pass@4": 0.992488178000693, + "pass@8": 0.9953082245624335, + "pass@16": 0.997132974983664, + "pass@32": 0.9981929377128227, + "pass@64": 0.9992150520076071, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881522147147147, + "pass@2": 0.9927101067603032, + "pass@4": 0.9944097843625403, + "pass@8": 0.995103475912635, + "pass@16": 0.9958460673407935, + "pass@32": 0.9966183345299415, + "pass@64": 0.9969772614942578, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9818880627009646, + "pass@2": 0.9903920709167786, + "pass@4": 0.9935343181579793, + "pass@8": 0.9947726177646687, + "pass@16": 0.9957678895375398, + "pass@32": 0.9970388159542295, + "pass@64": 0.9983478337657014, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9289434523809523, + "pass@2": 0.9530043119610053, + "pass@4": 0.9656035391409405, + "pass@8": 0.9720407256838862, + "pass@16": 0.9757086825588593, + "pass@32": 0.9784698585299846, + "pass@64": 0.9813627083401868, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8950449159021406, + "pass@2": 0.9291601952852221, + "pass@4": 0.9475928115063601, + "pass@8": 0.960967641143043, + "pass@16": 0.9716366374014452, + "pass@32": 0.9794922171298824, + "pass@64": 0.9848307287285822, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.7696035879629629, + "pass@2": 0.8292227902935744, + "pass@4": 0.8658306317034441, + "pass@8": 0.89026121313563, + "pass@16": 0.9075930135701733, + "pass@32": 0.9225986569963891, + "pass@64": 0.9382621076613159, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6916033011869436, + "pass@2": 0.7694590557022359, + "pass@4": 0.8216973023958051, + "pass@8": 0.8562871839082323, + "pass@16": 0.8799991156323995, + "pass@32": 0.8985396322425133, + "pass@64": 0.91363652395897, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.5952449845679012, + "pass@2": 0.6762832488820844, + "pass@4": 0.736550903937933, + "pass@8": 0.7814495701393551, + "pass@16": 0.8134452792685088, + "pass@32": 0.836684429948149, + "pass@64": 0.8557844677332779, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.4019174041297935, + "pass@2": 0.504326426149165, + "pass@4": 0.5959018344498971, + "pass@8": 0.6727407848216501, + "pass@16": 0.7339622740188452, + "pass@32": 0.7819087110814815, + "pass@64": 0.8193009236565792, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.31741810993975905, + "pass@2": 0.40953553386775415, + "pass@4": 0.48704305015336896, + "pass@8": 0.5499816842368038, + "pass@16": 0.604508368845186, + "pass@32": 0.6564436911909359, + "pass@64": 0.7013562136970772, + "pass@128": 0.7349397590361446 + }, + "18": { + "pass@1": 0.317741935483871, + "pass@2": 0.40378897320294627, + "pass@4": 0.47759907864339524, + "pass@8": 0.5356404084271387, + "pass@16": 0.581367839016435, + "pass@32": 0.6213430758174604, + "pass@64": 0.6621726039803202, + "pass@128": 0.7096774193548387 + }, + "19": { + "pass@1": 0.20104796072507553, + "pass@2": 0.277813667126579, + "pass@4": 0.3523299567727752, + "pass@8": 0.42098742880752116, + "pass@16": 0.48204430116010694, + "pass@32": 0.5352011641883591, + "pass@64": 0.5813005114434726, + "pass@128": 0.6193353474320241 + }, + "20": { + "pass@1": 0.1936233108108108, + "pass@2": 0.268572952874134, + "pass@4": 0.34309407523383884, + "pass@8": 0.412354850640087, + "pass@16": 0.4758931598412825, + "pass@32": 0.5322037216710581, + "pass@64": 0.5776411084860688, + "pass@128": 0.6126126126126126 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f12d9423082d7f0d495d353390fa02bc2ec83904 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db51a33c50b376a30a386b47b9722bff5da6403cf2689b2a4b17fda6abdd059f +size 1789648327 diff --git a/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_metrics.json b/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..111712b8b8716771d8fd395dbf8508dcd7d34ab5 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op11-14", + "id": { + "answer_accuracy": 0.9928567708333333, + "avg_response_len": 199.79739930555556, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9928567708333333, + "pass@2": 0.9948959700349949, + "pass@4": 0.9962162588843063, + "pass@8": 0.996999282682993, + "pass@16": 0.9974741171145197, + "pass@32": 0.9978465770581726, + "pass@64": 0.9982309070418071, + "pass@128": 0.9986666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9539296875, + "pass@2": 0.9635426919291339, + "pass@4": 0.9707023041807276, + "pass@8": 0.975490160066206, + "pass@16": 0.9789009518866644, + "pass@32": 0.9818573785913061, + "pass@64": 0.9851069875075766, + "pass@128": 0.989 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99925, + "pass@2": 0.9996391486220473, + "pass@4": 0.9998796986314211, + "pass@8": 0.9999868113249217, + "pass@16": 0.9999998839756833, + "pass@32": 0.9999999999981759, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956640625, + "pass@2": 0.9981594488188975, + "pass@4": 0.9994434281964756, + "pass@8": 0.9998959286259774, + "pass@16": 0.9999928848842898, + "pass@32": 0.9999999799014614, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999015625, + "pass@2": 0.9996764271653543, + "pass@4": 0.9999498568616423, + "pass@8": 0.9999984146505028, + "pass@16": 0.9999999988243893, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997109375, + "pass@2": 0.9985327263779529, + "pass@4": 0.999450328927634, + "pass@8": 0.999885866296035, + "pass@16": 0.9999927570983452, + "pass@32": 0.9999999799000695, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948828125, + "pass@2": 0.9972484005905513, + "pass@4": 0.9986864627859019, + "pass@8": 0.9995349095660551, + "pass@16": 0.9998925640782925, + "pass@32": 0.9999928988888469, + "pass@64": 0.9999999947663261, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959140625, + "pass@2": 0.9972661171259843, + "pass@4": 0.9978342508436445, + "pass@8": 0.9982014536172431, + "pass@16": 0.9984880132830111, + "pass@32": 0.9987689562436927, + "pass@64": 0.9989711811023622, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9539296875, + "2": 1.0, + "3": 0.9999453125, + "4": 0.99925, + "5": 0.9956640625, + "6": 0.999015625, + "7": 0.997109375, + "8": 0.9948828125, + "9": 0.9959140625 + }, + "per_op_avg_response_len": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376539, + "answer_accuracy": 0.9934856256332321, + "avg_response_len": 190.24900002110775, + "resp_tokens_sum": 72105893, + "pass_at_k": { + "pass@1": 0.9934856256332321, + "pass@2": 0.9952370897121903, + "pass@4": 0.9963667890044646, + "pass@8": 0.99714430403211, + "pass@16": 0.9976664856081482, + "pass@32": 0.9979998351197172, + "pass@64": 0.9983126058744111, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9555217551622419, + "pass@2": 0.9649799375188721, + "pass@4": 0.9725824357685375, + "pass@8": 0.9783204632700988, + "pass@16": 0.9825890111220292, + "pass@32": 0.9854794451027657, + "pass@64": 0.9882112861183845, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99765625, + "pass@2": 0.9988723394438976, + "pass@4": 0.9996240582231908, + "pass@8": 0.9999587853903806, + "pass@16": 0.99999963742401, + "pass@32": 0.9999999999942999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956597222222222, + "pass@2": 0.997699506311711, + "pass@4": 0.9989479258545063, + "pass@8": 0.999704562830816, + "pass@16": 0.9999774766429096, + "pass@32": 0.9999999361951407, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.999648741438755, + "pass@4": 0.999965682636564, + "pass@8": 0.9999996999237872, + "pass@16": 0.999999999991595, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986227964743589, + "pass@2": 0.9999313862810418, + "pass@4": 0.9999997707618278, + "pass@8": 0.9999999999988657, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9971675030497948, + "pass@4": 0.9971830761471716, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387419, + "answer_accuracy": 0.9917139375819135, + "avg_response_len": 197.73270089285714, + "resp_tokens_sum": 77245466, + "pass_at_k": { + "pass@1": 0.9917139375819135, + "pass@2": 0.9937211609658313, + "pass@4": 0.9950361093870473, + "pass@8": 0.9958125968364286, + "pass@16": 0.9963626120368996, + "pass@32": 0.9969249335365784, + "pass@64": 0.9974844614531034, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392973542944786, + "pass@2": 0.9498639105598762, + "pass@4": 0.9579157637887292, + "pass@8": 0.9633439750573325, + "pass@16": 0.967524339629329, + "pass@32": 0.9719200985373748, + "pass@64": 0.9765380222469652, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988224637681159, + "pass@2": 0.9997874586328882, + "pass@4": 0.9999922303190363, + "pass@8": 0.9999999910428016, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985449498567335, + "pass@2": 0.9996760288112266, + "pass@4": 0.9999785661641866, + "pass@8": 0.9999999015825531, + "pass@16": 0.9999999999993172, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960711705202312, + "pass@2": 0.9974437184242867, + "pass@4": 0.9988051376164685, + "pass@8": 0.9997047707744012, + "pass@16": 0.9999793315061124, + "pass@32": 0.9999999419107679, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9983739048464012, + "pass@4": 0.9996422492963027, + "pass@8": 0.9999796729938163, + "pass@16": 0.999999939849747, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945499625748503, + "pass@2": 0.9964954441510679, + "pass@4": 0.9973660230158355, + "pass@8": 0.9978422530337244, + "pass@16": 0.9984818259578424, + "pass@32": 0.9993082788304156, + "pass@64": 0.999913715875336, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379813, + "answer_accuracy": 0.9934010922330098, + "avg_response_len": 211.37231388098425, + "resp_tokens_sum": 80815245, + "pass_at_k": { + "pass@1": 0.9934010922330098, + "pass@2": 0.9957581936422663, + "pass@4": 0.9972728697245619, + "pass@8": 0.9980680329303928, + "pass@16": 0.9984191155702485, + "pass@32": 0.9986363523871554, + "pass@64": 0.9989126083117708, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9665578358208955, + "pass@2": 0.9753995769185567, + "pass@4": 0.9812427446569179, + "pass@8": 0.984445931787918, + "pass@16": 0.9862398280511495, + "pass@32": 0.9878623718754762, + "pass@64": 0.9903043768983087, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999964130759167, + "pass@4": 0.9999999986335528, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9924632352941176, + "pass@2": 0.9969336208893007, + "pass@4": 0.9993456238007014, + "pass@8": 0.9999676306603055, + "pass@16": 0.9999999404169826, + "pass@32": 0.9999999999999771, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9984966482649842, + "pass@2": 0.9993359404108397, + "pass@4": 0.9998654172565968, + "pass@8": 0.9999951072498166, + "pass@16": 0.9999999962922006, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966871044303798, + "pass@2": 0.9985314088507925, + "pass@4": 0.9996055398138525, + "pass@8": 0.9999623956767472, + "pass@16": 0.999999710126808, + "pass@32": 0.9999999999966581, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9901229354354354, + "pass@2": 0.9935347503014826, + "pass@4": 0.9964370444685404, + "pass@8": 0.9986250019627753, + "pass@16": 0.9996774337286254, + "pass@32": 0.9999786753419787, + "pass@64": 0.9999999842832614, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965082395498392, + "pass@2": 0.9982063511152746, + "pass@4": 0.9990804087591943, + "pass@8": 0.9997496498520251, + "pass@16": 0.9999841910388799, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.691875, + "avg_response_len": 372.99815546875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.691875, + "pass@2": 0.7569655265747888, + "pass@4": 0.8055117342988305, + "pass@8": 0.8415277440961516, + "pass@16": 0.8685987369738101, + "pass@32": 0.8897270269295073, + "pass@64": 0.9066655363534859, + "pass@128": 0.9202 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9625703125, + "pass@2": 0.9716028543307089, + "pass@4": 0.977065253562055, + "pass@8": 0.9805981358009112, + "pass@16": 0.9828878559740792, + "pass@32": 0.9848897753040814, + "pass@64": 0.9870662823610318, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.91546875, + "pass@2": 0.9322356053149612, + "pass@4": 0.9421446911323584, + "pass@8": 0.9489884448415973, + "pass@16": 0.9547641910879968, + "pass@32": 0.9604217627862317, + "pass@64": 0.9663588439701425, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.85984375, + "pass@2": 0.8906690452755908, + "pass@4": 0.9091363452380952, + "pass@8": 0.9214996283577351, + "pass@16": 0.9320471191065713, + "pass@32": 0.9430140808713036, + "pass@64": 0.953760160490955, + "pass@128": 0.964 + }, + "14": { + "pass@1": 0.866953125, + "pass@2": 0.9020903051181108, + "pass@4": 0.9213187425009368, + "pass@8": 0.9334330127351448, + "pass@16": 0.9427593873254092, + "pass@32": 0.9511044705782022, + "pass@64": 0.9583887065127129, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.82159375, + "pass@2": 0.87338939468504, + "pass@4": 0.9061740229658797, + "pass@8": 0.9284084065232308, + "pass@16": 0.9437730707115572, + "pass@32": 0.9545539010961099, + "pass@64": 0.9619371784947097, + "pass@128": 0.967 + }, + "16": { + "pass@1": 0.66428125, + "pass@2": 0.7492256397637792, + "pass@4": 0.8064780830521187, + "pass@8": 0.8457528599909511, + "pass@16": 0.872940244032758, + "pass@32": 0.8926041664005118, + "pass@64": 0.9079683868720971, + "pass@128": 0.92 + }, + "17": { + "pass@1": 0.603, + "pass@2": 0.6990440452755908, + "pass@4": 0.7653448331458559, + "pass@8": 0.8106834321313242, + "pass@16": 0.8440055876598439, + "pass@32": 0.8700455778571285, + "pass@64": 0.8908740562835517, + "pass@128": 0.908 + }, + "18": { + "pass@1": 0.53840625, + "pass@2": 0.6397306840551189, + "pass@4": 0.7131535387139104, + "pass@8": 0.7656584292259977, + "pass@16": 0.805760181584611, + "pass@32": 0.838258166150287, + "pass@64": 0.8651842279438802, + "pass@128": 0.887 + }, + "19": { + "pass@1": 0.3564453125, + "pass@2": 0.4690118110236226, + "pass@4": 0.5694118413010871, + "pass@8": 0.6518199874635647, + "pass@16": 0.7169103192164673, + "pass@32": 0.7679376199362757, + "pass@64": 0.8073694336016357, + "pass@128": 0.838 + }, + "20": { + "pass@1": 0.3301875, + "pass@2": 0.4426558809055117, + "pass@4": 0.5448899913760785, + "pass@8": 0.6284351038911395, + "pass@16": 0.6901394130387092, + "pass@32": 0.7344407483148729, + "pass@64": 0.7677480870040734, + "pass@128": 0.793 + } + }, + "per_op_accuracy": { + "11": 0.9625703125, + "12": 0.91546875, + "13": 0.85984375, + "14": 0.866953125, + "15": 0.82159375, + "16": 0.66428125, + "17": 0.603, + "18": 0.53840625, + "19": 0.3564453125, + "20": 0.3301875 + }, + "per_op_avg_response_len": { + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 300147, + "answer_accuracy": 0.6968494613670134, + "avg_response_len": 355.2223834509658, + "resp_tokens_sum": 153001385, + "pass_at_k": { + "pass@1": 0.6968494613670134, + "pass@2": 0.7615124940038193, + "pass@4": 0.8079247967425347, + "pass@8": 0.8413539091928377, + "pass@16": 0.8668610172246366, + "pass@32": 0.8874146419722095, + "pass@64": 0.9039114091381322, + "pass@128": 0.9170876671619613 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9595432011331445, + "pass@2": 0.9712151608262141, + "pass@4": 0.9783531306993142, + "pass@8": 0.983326614921076, + "pass@16": 0.9865718492617972, + "pass@32": 0.9888296194765508, + "pass@64": 0.9906259618168388, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.906007238700565, + "pass@2": 0.9247933487922062, + "pass@4": 0.9357417314361123, + "pass@8": 0.9421184863804423, + "pass@16": 0.9467630723566625, + "pass@32": 0.9518291711617118, + "pass@64": 0.9579773195275365, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8717672413793104, + "pass@2": 0.9035996161725869, + "pass@4": 0.9226742165654058, + "pass@8": 0.9340267016365622, + "pass@16": 0.9431541830177079, + "pass@32": 0.9532836501149419, + "pass@64": 0.9644626458159502, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8635854750778816, + "pass@2": 0.8954251447249002, + "pass@4": 0.9113720040251972, + "pass@8": 0.9206645009701866, + "pass@16": 0.9286928898625472, + "pass@32": 0.9369032185178561, + "pass@64": 0.9440107739042536, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.8376986994219653, + "pass@2": 0.8865166014291564, + "pass@4": 0.9181329861765831, + "pass@8": 0.9412534150029087, + "pass@16": 0.9562649099753139, + "pass@32": 0.9651885106215669, + "pass@64": 0.9706079061795896, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.6686743951612903, + "pass@2": 0.7454768065786131, + "pass@4": 0.7934225168426517, + "pass@8": 0.8257491148636001, + "pass@16": 0.8513642598353691, + "pass@32": 0.8730630130473881, + "pass@64": 0.8910014008528852, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.6197841401734104, + "pass@2": 0.7161787839652269, + "pass@4": 0.7794292514375006, + "pass@8": 0.8202572762642997, + "pass@16": 0.8505096438381178, + "pass@32": 0.8753376105726489, + "pass@64": 0.8944587031637984, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5136607142857142, + "pass@2": 0.6187155511811023, + "pass@4": 0.6944143609727348, + "pass@8": 0.7489642516122974, + "pass@16": 0.7936353786265803, + "pass@32": 0.8323554499320751, + "pass@64": 0.8633710891209031, + "pass@128": 0.8857142857142857 + }, + "19": { + "pass@1": 0.37058933933933935, + "pass@2": 0.48287460689035483, + "pass@4": 0.5762031899728748, + "pass@8": 0.6492350344930701, + "pass@16": 0.706125200061887, + "pass@32": 0.7499946710222914, + "pass@64": 0.7842006467978939, + "pass@128": 0.8138138138138138 + }, + "20": { + "pass@1": 0.34649493243243246, + "pass@2": 0.46130899600387765, + "pass@4": 0.5613634281075223, + "pass@8": 0.6403788685222684, + "pass@16": 0.6985517724335117, + "pass@32": 0.7410158124161551, + "pass@64": 0.7727139219338052, + "pass@128": 0.7957957957957958 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 302380, + "answer_accuracy": 0.7024513083556349, + "avg_response_len": 368.4694585377639, + "resp_tokens_sum": 158612837, + "pass_at_k": { + "pass@1": 0.7024513083556349, + "pass@2": 0.7656160003722866, + "pass@4": 0.8130046151122302, + "pass@8": 0.8491108703695945, + "pass@16": 0.8761795320620611, + "pass@32": 0.8966491248930549, + "pass@64": 0.9127342481802431, + "pass@128": 0.9259589652096343 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9681852409638554, + "pass@2": 0.9741481921781614, + "pass@4": 0.9772397477988446, + "pass@8": 0.9789669965699488, + "pass@16": 0.9799547315679026, + "pass@32": 0.9809906007078771, + "pass@64": 0.9826866521200436, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9085276253918495, + "pass@2": 0.9235974409448813, + "pass@4": 0.9330263857104065, + "pass@8": 0.9409230257953107, + "pass@16": 0.9477558425747747, + "pass@32": 0.9527120358305108, + "pass@64": 0.95642424625744, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8668154761904762, + "pass@2": 0.8926307235492627, + "pass@4": 0.9088578126788774, + "pass@8": 0.9216598215625059, + "pass@16": 0.9320716043391158, + "pass@32": 0.941360763388727, + "pass@64": 0.950378253308469, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.881281980994152, + "pass@2": 0.9201054329557498, + "pass@4": 0.9419351528427365, + "pass@8": 0.9547325691053806, + "pass@16": 0.9625703344338206, + "pass@32": 0.9680469863909118, + "pass@64": 0.9725173602632785, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8279829545454546, + "pass@2": 0.8793225065616795, + "pass@4": 0.9134771270068515, + "pass@8": 0.9379655607421947, + "pass@16": 0.9560091862643495, + "pass@32": 0.9682017448322129, + "pass@64": 0.9747783770514524, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6672008547008547, + "pass@2": 0.7537498317517997, + "pass@4": 0.8141053287142518, + "pass@8": 0.8573749750163748, + "pass@16": 0.886024033601711, + "pass@32": 0.9049191024077768, + "pass@64": 0.9190486363240371, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.6349718555900621, + "pass@2": 0.7272980143786375, + "pass@4": 0.7871718557501735, + "pass@8": 0.8293996571531411, + "pass@16": 0.8622377160167927, + "pass@32": 0.8877773954413081, + "pass@64": 0.9070769693130836, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.5722426470588236, + "pass@2": 0.6690550457387674, + "pass@4": 0.7391316479741499, + "pass@8": 0.7903393556142831, + "pass@16": 0.8293274169491589, + "pass@32": 0.8609394488494504, + "pass@64": 0.8900216102839796, + "pass@128": 0.9176470588235294 + }, + "19": { + "pass@1": 0.36755952380952384, + "pass@2": 0.4771494627624668, + "pass@4": 0.5758954558693551, + "pass@8": 0.6572878262769214, + "pass@16": 0.7219972024993643, + "pass@32": 0.7752524575772055, + "pass@64": 0.8196462374053157, + "pass@128": 0.8541666666666666 + }, + "20": { + "pass@1": 0.3302067739520958, + "pass@2": 0.4390511545806021, + "pass@4": 0.5388564448717861, + "pass@8": 0.6219737802822332, + "pass@16": 0.6834840446587457, + "pass@32": 0.7260101372704013, + "pass@64": 0.7543047807447868, + "pass@128": 0.7724550898203593 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 283073, + "answer_accuracy": 0.6758886957518337, + "avg_response_len": 395.9338158045538, + "resp_tokens_sum": 165823417, + "pass_at_k": { + "pass@1": 0.6758886957518337, + "pass@2": 0.7433982622778896, + "pass@4": 0.7953288145865972, + "pass@8": 0.8339124937270136, + "pass@16": 0.8625942177421915, + "pass@32": 0.8849905232405783, + "pass@64": 0.903260450841926, + "pass@128": 0.9174816625916871 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9600446428571429, + "pass@2": 0.9693546119235098, + "pass@4": 0.9754381020824777, + "pass@8": 0.9792596757858676, + "pass@16": 0.9818508644575293, + "pass@32": 0.9845842545836299, + "pass@64": 0.9876931724947077, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9324827981651376, + "pass@2": 0.9487191932143805, + "pass@4": 0.9579715631876291, + "pass@8": 0.964293747505247, + "pass@16": 0.9702628430959813, + "pass@32": 0.9772449442356358, + "pass@64": 0.9851239703402783, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8404224537037037, + "pass@2": 0.8757765413871874, + "pass@4": 0.8961142932943567, + "pass@8": 0.908989364931657, + "pass@16": 0.9210844814038844, + "pass@32": 0.9347247035643859, + "pass@64": 0.9469491976683431, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8556194362017804, + "pass@2": 0.8901566041262641, + "pass@4": 0.9098709404647859, + "pass@8": 0.9239797308298969, + "pass@16": 0.9360531611369882, + "pass@32": 0.9474375907664061, + "pass@64": 0.957745729612481, + "pass@128": 0.9673590504451038 + }, + "15": { + "pass@1": 0.7978877314814815, + "pass@2": 0.8533278500777681, + "pass@4": 0.8859646846806184, + "pass@8": 0.9049570675533932, + "pass@16": 0.9179703098545164, + "pass@32": 0.9292966068716593, + "pass@64": 0.939598699165408, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6572409660766961, + "pass@2": 0.7479694359278098, + "pass@4": 0.8105195647557326, + "pass@8": 0.8520118529571892, + "pass@16": 0.879123562506178, + "pass@32": 0.8977227947218035, + "pass@64": 0.9120114491385409, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5544992469879518, + "pass@2": 0.6537838114268096, + "pass@4": 0.7294969114365216, + "pass@8": 0.7825533883751986, + "pass@16": 0.8195442961880983, + "pass@32": 0.8473326604424434, + "pass@64": 0.871423376114652, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.5292338709677419, + "pass@2": 0.6312952438404874, + "pass@4": 0.7058185550394909, + "pass@8": 0.7574372911381855, + "pass@16": 0.7936015396857868, + "pass@32": 0.8200462776298317, + "pass@64": 0.8399903201774559, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.3309337235649547, + "pass@2": 0.4468046750481719, + "pass@4": 0.5559979029245206, + "pass@8": 0.6488701249194181, + "pass@16": 0.7225968808340559, + "pass@32": 0.7785636517822091, + "pass@64": 0.8182159590626885, + "pass@128": 0.8459214501510574 + }, + "20": { + "pass@1": 0.31386073573573575, + "pass@2": 0.4276183171360337, + "pass@4": 0.5344682199071957, + "pass@8": 0.6229720662431183, + "pass@16": 0.6884024081151023, + "pass@32": 0.7363216124083445, + "pass@64": 0.776265928622691, + "pass@128": 0.8108108108108109 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8344453125, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8344453125, + "pass@2": 0.8696694208454039, + "pass@4": 0.8958454564708882, + "pass@8": 0.9151721571109854, + "pass@16": 0.9296449696720248, + "pass@32": 0.9409415506746711, + "pass@64": 0.9500386066795303, + "pass@128": 0.9573684210526315 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9539296875, + "pass@2": 0.9635426919291339, + "pass@4": 0.9707023041807276, + "pass@8": 0.975490160066206, + "pass@16": 0.9789009518866644, + "pass@32": 0.9818573785913061, + "pass@64": 0.9851069875075766, + "pass@128": 0.989 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99925, + "pass@2": 0.9996391486220473, + "pass@4": 0.9998796986314211, + "pass@8": 0.9999868113249217, + "pass@16": 0.9999998839756833, + "pass@32": 0.9999999999981759, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956640625, + "pass@2": 0.9981594488188975, + "pass@4": 0.9994434281964756, + "pass@8": 0.9998959286259774, + "pass@16": 0.9999928848842898, + "pass@32": 0.9999999799014614, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999015625, + "pass@2": 0.9996764271653543, + "pass@4": 0.9999498568616423, + "pass@8": 0.9999984146505028, + "pass@16": 0.9999999988243893, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997109375, + "pass@2": 0.9985327263779529, + "pass@4": 0.999450328927634, + "pass@8": 0.999885866296035, + "pass@16": 0.9999927570983452, + "pass@32": 0.9999999799000695, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948828125, + "pass@2": 0.9972484005905513, + "pass@4": 0.9986864627859019, + "pass@8": 0.9995349095660551, + "pass@16": 0.9998925640782925, + "pass@32": 0.9999928988888469, + "pass@64": 0.9999999947663261, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959140625, + "pass@2": 0.9972661171259843, + "pass@4": 0.9978342508436445, + "pass@8": 0.9982014536172431, + "pass@16": 0.9984880132830111, + "pass@32": 0.9987689562436927, + "pass@64": 0.9989711811023622, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9625703125, + "pass@2": 0.9716028543307089, + "pass@4": 0.977065253562055, + "pass@8": 0.9805981358009112, + "pass@16": 0.9828878559740792, + "pass@32": 0.9848897753040814, + "pass@64": 0.9870662823610318, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.91546875, + "pass@2": 0.9322356053149612, + "pass@4": 0.9421446911323584, + "pass@8": 0.9489884448415973, + "pass@16": 0.9547641910879968, + "pass@32": 0.9604217627862317, + "pass@64": 0.9663588439701425, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.85984375, + "pass@2": 0.8906690452755908, + "pass@4": 0.9091363452380952, + "pass@8": 0.9214996283577351, + "pass@16": 0.9320471191065713, + "pass@32": 0.9430140808713036, + "pass@64": 0.953760160490955, + "pass@128": 0.964 + }, + "14": { + "pass@1": 0.866953125, + "pass@2": 0.9020903051181108, + "pass@4": 0.9213187425009368, + "pass@8": 0.9334330127351448, + "pass@16": 0.9427593873254092, + "pass@32": 0.9511044705782022, + "pass@64": 0.9583887065127129, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.82159375, + "pass@2": 0.87338939468504, + "pass@4": 0.9061740229658797, + "pass@8": 0.9284084065232308, + "pass@16": 0.9437730707115572, + "pass@32": 0.9545539010961099, + "pass@64": 0.9619371784947097, + "pass@128": 0.967 + }, + "16": { + "pass@1": 0.66428125, + "pass@2": 0.7492256397637792, + "pass@4": 0.8064780830521187, + "pass@8": 0.8457528599909511, + "pass@16": 0.872940244032758, + "pass@32": 0.8926041664005118, + "pass@64": 0.9079683868720971, + "pass@128": 0.92 + }, + "17": { + "pass@1": 0.603, + "pass@2": 0.6990440452755908, + "pass@4": 0.7653448331458559, + "pass@8": 0.8106834321313242, + "pass@16": 0.8440055876598439, + "pass@32": 0.8700455778571285, + "pass@64": 0.8908740562835517, + "pass@128": 0.908 + }, + "18": { + "pass@1": 0.53840625, + "pass@2": 0.6397306840551189, + "pass@4": 0.7131535387139104, + "pass@8": 0.7656584292259977, + "pass@16": 0.805760181584611, + "pass@32": 0.838258166150287, + "pass@64": 0.8651842279438802, + "pass@128": 0.887 + }, + "19": { + "pass@1": 0.3564453125, + "pass@2": 0.4690118110236226, + "pass@4": 0.5694118413010871, + "pass@8": 0.6518199874635647, + "pass@16": 0.7169103192164673, + "pass@32": 0.7679376199362757, + "pass@64": 0.8073694336016357, + "pass@128": 0.838 + }, + "20": { + "pass@1": 0.3301875, + "pass@2": 0.4426558809055117, + "pass@4": 0.5448899913760785, + "pass@8": 0.6284351038911395, + "pass@16": 0.6901394130387092, + "pass@32": 0.7344407483148729, + "pass@64": 0.7677480870040734, + "pass@128": 0.793 + } + }, + "per_op_accuracy": { + "10": 0.9539296875, + "2": 1.0, + "3": 0.9999453125, + "4": 0.99925, + "5": 0.9956640625, + "6": 0.999015625, + "7": 0.997109375, + "8": 0.9948828125, + "9": 0.9959140625, + "11": 0.9625703125, + "12": 0.91546875, + "13": 0.85984375, + "14": 0.866953125, + "15": 0.82159375, + "16": 0.66428125, + "17": 0.603, + "18": 0.53840625, + "19": 0.3564453125, + "20": 0.3301875 + }, + "per_op_avg_response_len": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875, + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875, + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 676686, + "answer_accuracy": 0.8356954434081568, + "avg_response_len": 278.0035740396775, + "resp_tokens_sum": 225107278, + "pass_at_k": { + "pass@1": 0.8356954434081568, + "pass@2": 0.8709115657541193, + "pass@4": 0.8961285177491005, + "pass@8": 0.9142744528411244, + "pass@16": 0.9280869090810386, + "pass@32": 0.939176064183682, + "pass@64": 0.948097615830537, + "pass@128": 0.9552639898830224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9555217551622419, + "pass@2": 0.9649799375188721, + "pass@4": 0.9725824357685375, + "pass@8": 0.9783204632700988, + "pass@16": 0.9825890111220292, + "pass@32": 0.9854794451027657, + "pass@64": 0.9882112861183845, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99765625, + "pass@2": 0.9988723394438976, + "pass@4": 0.9996240582231908, + "pass@8": 0.9999587853903806, + "pass@16": 0.99999963742401, + "pass@32": 0.9999999999942999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956597222222222, + "pass@2": 0.997699506311711, + "pass@4": 0.9989479258545063, + "pass@8": 0.999704562830816, + "pass@16": 0.9999774766429096, + "pass@32": 0.9999999361951407, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.999648741438755, + "pass@4": 0.999965682636564, + "pass@8": 0.9999996999237872, + "pass@16": 0.999999999991595, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986227964743589, + "pass@2": 0.9999313862810418, + "pass@4": 0.9999997707618278, + "pass@8": 0.9999999999988657, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9971675030497948, + "pass@4": 0.9971830761471716, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9595432011331445, + "pass@2": 0.9712151608262141, + "pass@4": 0.9783531306993142, + "pass@8": 0.983326614921076, + "pass@16": 0.9865718492617972, + "pass@32": 0.9888296194765508, + "pass@64": 0.9906259618168388, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.906007238700565, + "pass@2": 0.9247933487922062, + "pass@4": 0.9357417314361123, + "pass@8": 0.9421184863804423, + "pass@16": 0.9467630723566625, + "pass@32": 0.9518291711617118, + "pass@64": 0.9579773195275365, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8717672413793104, + "pass@2": 0.9035996161725869, + "pass@4": 0.9226742165654058, + "pass@8": 0.9340267016365622, + "pass@16": 0.9431541830177079, + "pass@32": 0.9532836501149419, + "pass@64": 0.9644626458159502, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8635854750778816, + "pass@2": 0.8954251447249002, + "pass@4": 0.9113720040251972, + "pass@8": 0.9206645009701866, + "pass@16": 0.9286928898625472, + "pass@32": 0.9369032185178561, + "pass@64": 0.9440107739042536, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.8376986994219653, + "pass@2": 0.8865166014291564, + "pass@4": 0.9181329861765831, + "pass@8": 0.9412534150029087, + "pass@16": 0.9562649099753139, + "pass@32": 0.9651885106215669, + "pass@64": 0.9706079061795896, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.6686743951612903, + "pass@2": 0.7454768065786131, + "pass@4": 0.7934225168426517, + "pass@8": 0.8257491148636001, + "pass@16": 0.8513642598353691, + "pass@32": 0.8730630130473881, + "pass@64": 0.8910014008528852, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.6197841401734104, + "pass@2": 0.7161787839652269, + "pass@4": 0.7794292514375006, + "pass@8": 0.8202572762642997, + "pass@16": 0.8505096438381178, + "pass@32": 0.8753376105726489, + "pass@64": 0.8944587031637984, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5136607142857142, + "pass@2": 0.6187155511811023, + "pass@4": 0.6944143609727348, + "pass@8": 0.7489642516122974, + "pass@16": 0.7936353786265803, + "pass@32": 0.8323554499320751, + "pass@64": 0.8633710891209031, + "pass@128": 0.8857142857142857 + }, + "19": { + "pass@1": 0.37058933933933935, + "pass@2": 0.48287460689035483, + "pass@4": 0.5762031899728748, + "pass@8": 0.6492350344930701, + "pass@16": 0.706125200061887, + "pass@32": 0.7499946710222914, + "pass@64": 0.7842006467978939, + "pass@128": 0.8138138138138138 + }, + "20": { + "pass@1": 0.34649493243243246, + "pass@2": 0.46130899600387765, + "pass@4": 0.5613634281075223, + "pass@8": 0.6403788685222684, + "pass@16": 0.6985517724335117, + "pass@32": 0.7410158124161551, + "pass@64": 0.7727139219338052, + "pass@128": 0.7957957957957958 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689799, + "answer_accuracy": 0.8400708787996882, + "avg_response_len": 287.2397493667186, + "resp_tokens_sum": 235858303, + "pass_at_k": { + "pass@1": 0.8400708787996882, + "pass@2": 0.8741392973530203, + "pass@4": 0.8996079074780458, + "pass@8": 0.9189056746060298, + "pass@16": 0.9333578266970128, + "pass@32": 0.9443563373607163, + "pass@64": 0.9530550043624433, + "pass@128": 0.9602494154325799 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392973542944786, + "pass@2": 0.9498639105598762, + "pass@4": 0.9579157637887292, + "pass@8": 0.9633439750573325, + "pass@16": 0.967524339629329, + "pass@32": 0.9719200985373748, + "pass@64": 0.9765380222469652, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988224637681159, + "pass@2": 0.9997874586328882, + "pass@4": 0.9999922303190363, + "pass@8": 0.9999999910428016, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985449498567335, + "pass@2": 0.9996760288112266, + "pass@4": 0.9999785661641866, + "pass@8": 0.9999999015825531, + "pass@16": 0.9999999999993172, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960711705202312, + "pass@2": 0.9974437184242867, + "pass@4": 0.9988051376164685, + "pass@8": 0.9997047707744012, + "pass@16": 0.9999793315061124, + "pass@32": 0.9999999419107679, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9983739048464012, + "pass@4": 0.9996422492963027, + "pass@8": 0.9999796729938163, + "pass@16": 0.999999939849747, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945499625748503, + "pass@2": 0.9964954441510679, + "pass@4": 0.9973660230158355, + "pass@8": 0.9978422530337244, + "pass@16": 0.9984818259578424, + "pass@32": 0.9993082788304156, + "pass@64": 0.999913715875336, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9681852409638554, + "pass@2": 0.9741481921781614, + "pass@4": 0.9772397477988446, + "pass@8": 0.9789669965699488, + "pass@16": 0.9799547315679026, + "pass@32": 0.9809906007078771, + "pass@64": 0.9826866521200436, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9085276253918495, + "pass@2": 0.9235974409448813, + "pass@4": 0.9330263857104065, + "pass@8": 0.9409230257953107, + "pass@16": 0.9477558425747747, + "pass@32": 0.9527120358305108, + "pass@64": 0.95642424625744, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8668154761904762, + "pass@2": 0.8926307235492627, + "pass@4": 0.9088578126788774, + "pass@8": 0.9216598215625059, + "pass@16": 0.9320716043391158, + "pass@32": 0.941360763388727, + "pass@64": 0.950378253308469, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.881281980994152, + "pass@2": 0.9201054329557498, + "pass@4": 0.9419351528427365, + "pass@8": 0.9547325691053806, + "pass@16": 0.9625703344338206, + "pass@32": 0.9680469863909118, + "pass@64": 0.9725173602632785, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8279829545454546, + "pass@2": 0.8793225065616795, + "pass@4": 0.9134771270068515, + "pass@8": 0.9379655607421947, + "pass@16": 0.9560091862643495, + "pass@32": 0.9682017448322129, + "pass@64": 0.9747783770514524, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6672008547008547, + "pass@2": 0.7537498317517997, + "pass@4": 0.8141053287142518, + "pass@8": 0.8573749750163748, + "pass@16": 0.886024033601711, + "pass@32": 0.9049191024077768, + "pass@64": 0.9190486363240371, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.6349718555900621, + "pass@2": 0.7272980143786375, + "pass@4": 0.7871718557501735, + "pass@8": 0.8293996571531411, + "pass@16": 0.8622377160167927, + "pass@32": 0.8877773954413081, + "pass@64": 0.9070769693130836, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.5722426470588236, + "pass@2": 0.6690550457387674, + "pass@4": 0.7391316479741499, + "pass@8": 0.7903393556142831, + "pass@16": 0.8293274169491589, + "pass@32": 0.8609394488494504, + "pass@64": 0.8900216102839796, + "pass@128": 0.9176470588235294 + }, + "19": { + "pass@1": 0.36755952380952384, + "pass@2": 0.4771494627624668, + "pass@4": 0.5758954558693551, + "pass@8": 0.6572878262769214, + "pass@16": 0.7219972024993643, + "pass@32": 0.7752524575772055, + "pass@64": 0.8196462374053157, + "pass@128": 0.8541666666666666 + }, + "20": { + "pass@1": 0.3302067739520958, + "pass@2": 0.4390511545806021, + "pass@4": 0.5388564448717861, + "pass@8": 0.6219737802822332, + "pass@16": 0.6834840446587457, + "pass@32": 0.7260101372704013, + "pass@64": 0.7543047807447868, + "pass@128": 0.7724550898203593 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 662886, + "answer_accuracy": 0.827416020929861, + "avg_response_len": 307.85501627656174, + "resp_tokens_sum": 246638662, + "pass_at_k": { + "pass@1": 0.827416020929861, + "pass@2": 0.8638326950922894, + "pass@4": 0.8917031383918479, + "pass@8": 0.9122528988397226, + "pass@16": 0.9274143119764798, + "pass@32": 0.9392260387639582, + "pass@64": 0.9489087963224284, + "pass@128": 0.9565425786866911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9665578358208955, + "pass@2": 0.9753995769185567, + "pass@4": 0.9812427446569179, + "pass@8": 0.984445931787918, + "pass@16": 0.9862398280511495, + "pass@32": 0.9878623718754762, + "pass@64": 0.9903043768983087, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999964130759167, + "pass@4": 0.9999999986335528, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9924632352941176, + "pass@2": 0.9969336208893007, + "pass@4": 0.9993456238007014, + "pass@8": 0.9999676306603055, + "pass@16": 0.9999999404169826, + "pass@32": 0.9999999999999771, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9984966482649842, + "pass@2": 0.9993359404108397, + "pass@4": 0.9998654172565968, + "pass@8": 0.9999951072498166, + "pass@16": 0.9999999962922006, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966871044303798, + "pass@2": 0.9985314088507925, + "pass@4": 0.9996055398138525, + "pass@8": 0.9999623956767472, + "pass@16": 0.999999710126808, + "pass@32": 0.9999999999966581, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9901229354354354, + "pass@2": 0.9935347503014826, + "pass@4": 0.9964370444685404, + "pass@8": 0.9986250019627753, + "pass@16": 0.9996774337286254, + "pass@32": 0.9999786753419787, + "pass@64": 0.9999999842832614, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965082395498392, + "pass@2": 0.9982063511152746, + "pass@4": 0.9990804087591943, + "pass@8": 0.9997496498520251, + "pass@16": 0.9999841910388799, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9600446428571429, + "pass@2": 0.9693546119235098, + "pass@4": 0.9754381020824777, + "pass@8": 0.9792596757858676, + "pass@16": 0.9818508644575293, + "pass@32": 0.9845842545836299, + "pass@64": 0.9876931724947077, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9324827981651376, + "pass@2": 0.9487191932143805, + "pass@4": 0.9579715631876291, + "pass@8": 0.964293747505247, + "pass@16": 0.9702628430959813, + "pass@32": 0.9772449442356358, + "pass@64": 0.9851239703402783, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8404224537037037, + "pass@2": 0.8757765413871874, + "pass@4": 0.8961142932943567, + "pass@8": 0.908989364931657, + "pass@16": 0.9210844814038844, + "pass@32": 0.9347247035643859, + "pass@64": 0.9469491976683431, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8556194362017804, + "pass@2": 0.8901566041262641, + "pass@4": 0.9098709404647859, + "pass@8": 0.9239797308298969, + "pass@16": 0.9360531611369882, + "pass@32": 0.9474375907664061, + "pass@64": 0.957745729612481, + "pass@128": 0.9673590504451038 + }, + "15": { + "pass@1": 0.7978877314814815, + "pass@2": 0.8533278500777681, + "pass@4": 0.8859646846806184, + "pass@8": 0.9049570675533932, + "pass@16": 0.9179703098545164, + "pass@32": 0.9292966068716593, + "pass@64": 0.939598699165408, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6572409660766961, + "pass@2": 0.7479694359278098, + "pass@4": 0.8105195647557326, + "pass@8": 0.8520118529571892, + "pass@16": 0.879123562506178, + "pass@32": 0.8977227947218035, + "pass@64": 0.9120114491385409, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5544992469879518, + "pass@2": 0.6537838114268096, + "pass@4": 0.7294969114365216, + "pass@8": 0.7825533883751986, + "pass@16": 0.8195442961880983, + "pass@32": 0.8473326604424434, + "pass@64": 0.871423376114652, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.5292338709677419, + "pass@2": 0.6312952438404874, + "pass@4": 0.7058185550394909, + "pass@8": 0.7574372911381855, + "pass@16": 0.7936015396857868, + "pass@32": 0.8200462776298317, + "pass@64": 0.8399903201774559, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.3309337235649547, + "pass@2": 0.4468046750481719, + "pass@4": 0.5559979029245206, + "pass@8": 0.6488701249194181, + "pass@16": 0.7225968808340559, + "pass@32": 0.7785636517822091, + "pass@64": 0.8182159590626885, + "pass@128": 0.8459214501510574 + }, + "20": { + "pass@1": 0.31386073573573575, + "pass@2": 0.4276183171360337, + "pass@4": 0.5344682199071957, + "pass@8": 0.6229720662431183, + "pass@16": 0.6884024081151023, + "pass@32": 0.7363216124083445, + "pass@64": 0.776265928622691, + "pass@128": 0.8108108108108109 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fca87d1bc0d3a628e1e39bd52034314adf757538 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c64d4df2fd43c45e1d60f1359f3a1d59022cb4d422649bd651cc6a09d6335862 +size 2700799216 diff --git a/results/composition/SDAR-100M-block4-epoch4-op11-14/summary.csv b/results/composition/SDAR-100M-block4-epoch4-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..98e9bac36dbed8b0fdc945c1f4a12b0cefe26f6c --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op11-14,0.9928567708333333,0.691875,0.8344453125,0.0,0.0,0.0,199.79739930555556,372.99815546875,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch4-op11-14/summary.json b/results/composition/SDAR-100M-block4-epoch4-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4897fdecf36111e8f967fbd6fa0c6e55d36a3421 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op11-14", + "id_acc": 0.9928567708333333, + "ood_acc": 0.691875, + "total_acc": 0.8344453125, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.79739930555556, + "ood_avg_resp_len": 372.99815546875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 676686, + "answer_accuracy": 0.8356954434081568, + "avg_response_len": 278.0035740396775, + "resp_tokens_sum": 225107278, + "pass_at_k": { + "pass@1": 0.8356954434081568, + "pass@2": 0.8709115657541193, + "pass@4": 0.8961285177491005, + "pass@8": 0.9142744528411244, + "pass@16": 0.9280869090810386, + "pass@32": 0.939176064183682, + "pass@64": 0.948097615830537, + "pass@128": 0.9552639898830224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9555217551622419, + "pass@2": 0.9649799375188721, + "pass@4": 0.9725824357685375, + "pass@8": 0.9783204632700988, + "pass@16": 0.9825890111220292, + "pass@32": 0.9854794451027657, + "pass@64": 0.9882112861183845, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99765625, + "pass@2": 0.9988723394438976, + "pass@4": 0.9996240582231908, + "pass@8": 0.9999587853903806, + "pass@16": 0.99999963742401, + "pass@32": 0.9999999999942999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956597222222222, + "pass@2": 0.997699506311711, + "pass@4": 0.9989479258545063, + "pass@8": 0.999704562830816, + "pass@16": 0.9999774766429096, + "pass@32": 0.9999999361951407, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.999648741438755, + "pass@4": 0.999965682636564, + "pass@8": 0.9999996999237872, + "pass@16": 0.999999999991595, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986227964743589, + "pass@2": 0.9999313862810418, + "pass@4": 0.9999997707618278, + "pass@8": 0.9999999999988657, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9971675030497948, + "pass@4": 0.9971830761471716, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9595432011331445, + "pass@2": 0.9712151608262141, + "pass@4": 0.9783531306993142, + "pass@8": 0.983326614921076, + "pass@16": 0.9865718492617972, + "pass@32": 0.9888296194765508, + "pass@64": 0.9906259618168388, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.906007238700565, + "pass@2": 0.9247933487922062, + "pass@4": 0.9357417314361123, + "pass@8": 0.9421184863804423, + "pass@16": 0.9467630723566625, + "pass@32": 0.9518291711617118, + "pass@64": 0.9579773195275365, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8717672413793104, + "pass@2": 0.9035996161725869, + "pass@4": 0.9226742165654058, + "pass@8": 0.9340267016365622, + "pass@16": 0.9431541830177079, + "pass@32": 0.9532836501149419, + "pass@64": 0.9644626458159502, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8635854750778816, + "pass@2": 0.8954251447249002, + "pass@4": 0.9113720040251972, + "pass@8": 0.9206645009701866, + "pass@16": 0.9286928898625472, + "pass@32": 0.9369032185178561, + "pass@64": 0.9440107739042536, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.8376986994219653, + "pass@2": 0.8865166014291564, + "pass@4": 0.9181329861765831, + "pass@8": 0.9412534150029087, + "pass@16": 0.9562649099753139, + "pass@32": 0.9651885106215669, + "pass@64": 0.9706079061795896, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.6686743951612903, + "pass@2": 0.7454768065786131, + "pass@4": 0.7934225168426517, + "pass@8": 0.8257491148636001, + "pass@16": 0.8513642598353691, + "pass@32": 0.8730630130473881, + "pass@64": 0.8910014008528852, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.6197841401734104, + "pass@2": 0.7161787839652269, + "pass@4": 0.7794292514375006, + "pass@8": 0.8202572762642997, + "pass@16": 0.8505096438381178, + "pass@32": 0.8753376105726489, + "pass@64": 0.8944587031637984, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5136607142857142, + "pass@2": 0.6187155511811023, + "pass@4": 0.6944143609727348, + "pass@8": 0.7489642516122974, + "pass@16": 0.7936353786265803, + "pass@32": 0.8323554499320751, + "pass@64": 0.8633710891209031, + "pass@128": 0.8857142857142857 + }, + "19": { + "pass@1": 0.37058933933933935, + "pass@2": 0.48287460689035483, + "pass@4": 0.5762031899728748, + "pass@8": 0.6492350344930701, + "pass@16": 0.706125200061887, + "pass@32": 0.7499946710222914, + "pass@64": 0.7842006467978939, + "pass@128": 0.8138138138138138 + }, + "20": { + "pass@1": 0.34649493243243246, + "pass@2": 0.46130899600387765, + "pass@4": 0.5613634281075223, + "pass@8": 0.6403788685222684, + "pass@16": 0.6985517724335117, + "pass@32": 0.7410158124161551, + "pass@64": 0.7727139219338052, + "pass@128": 0.7957957957957958 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689799, + "answer_accuracy": 0.8400708787996882, + "avg_response_len": 287.2397493667186, + "resp_tokens_sum": 235858303, + "pass_at_k": { + "pass@1": 0.8400708787996882, + "pass@2": 0.8741392973530203, + "pass@4": 0.8996079074780458, + "pass@8": 0.9189056746060298, + "pass@16": 0.9333578266970128, + "pass@32": 0.9443563373607163, + "pass@64": 0.9530550043624433, + "pass@128": 0.9602494154325799 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392973542944786, + "pass@2": 0.9498639105598762, + "pass@4": 0.9579157637887292, + "pass@8": 0.9633439750573325, + "pass@16": 0.967524339629329, + "pass@32": 0.9719200985373748, + "pass@64": 0.9765380222469652, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988224637681159, + "pass@2": 0.9997874586328882, + "pass@4": 0.9999922303190363, + "pass@8": 0.9999999910428016, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985449498567335, + "pass@2": 0.9996760288112266, + "pass@4": 0.9999785661641866, + "pass@8": 0.9999999015825531, + "pass@16": 0.9999999999993172, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960711705202312, + "pass@2": 0.9974437184242867, + "pass@4": 0.9988051376164685, + "pass@8": 0.9997047707744012, + "pass@16": 0.9999793315061124, + "pass@32": 0.9999999419107679, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9983739048464012, + "pass@4": 0.9996422492963027, + "pass@8": 0.9999796729938163, + "pass@16": 0.999999939849747, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945499625748503, + "pass@2": 0.9964954441510679, + "pass@4": 0.9973660230158355, + "pass@8": 0.9978422530337244, + "pass@16": 0.9984818259578424, + "pass@32": 0.9993082788304156, + "pass@64": 0.999913715875336, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9681852409638554, + "pass@2": 0.9741481921781614, + "pass@4": 0.9772397477988446, + "pass@8": 0.9789669965699488, + "pass@16": 0.9799547315679026, + "pass@32": 0.9809906007078771, + "pass@64": 0.9826866521200436, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9085276253918495, + "pass@2": 0.9235974409448813, + "pass@4": 0.9330263857104065, + "pass@8": 0.9409230257953107, + "pass@16": 0.9477558425747747, + "pass@32": 0.9527120358305108, + "pass@64": 0.95642424625744, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8668154761904762, + "pass@2": 0.8926307235492627, + "pass@4": 0.9088578126788774, + "pass@8": 0.9216598215625059, + "pass@16": 0.9320716043391158, + "pass@32": 0.941360763388727, + "pass@64": 0.950378253308469, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.881281980994152, + "pass@2": 0.9201054329557498, + "pass@4": 0.9419351528427365, + "pass@8": 0.9547325691053806, + "pass@16": 0.9625703344338206, + "pass@32": 0.9680469863909118, + "pass@64": 0.9725173602632785, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8279829545454546, + "pass@2": 0.8793225065616795, + "pass@4": 0.9134771270068515, + "pass@8": 0.9379655607421947, + "pass@16": 0.9560091862643495, + "pass@32": 0.9682017448322129, + "pass@64": 0.9747783770514524, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6672008547008547, + "pass@2": 0.7537498317517997, + "pass@4": 0.8141053287142518, + "pass@8": 0.8573749750163748, + "pass@16": 0.886024033601711, + "pass@32": 0.9049191024077768, + "pass@64": 0.9190486363240371, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.6349718555900621, + "pass@2": 0.7272980143786375, + "pass@4": 0.7871718557501735, + "pass@8": 0.8293996571531411, + "pass@16": 0.8622377160167927, + "pass@32": 0.8877773954413081, + "pass@64": 0.9070769693130836, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.5722426470588236, + "pass@2": 0.6690550457387674, + "pass@4": 0.7391316479741499, + "pass@8": 0.7903393556142831, + "pass@16": 0.8293274169491589, + "pass@32": 0.8609394488494504, + "pass@64": 0.8900216102839796, + "pass@128": 0.9176470588235294 + }, + "19": { + "pass@1": 0.36755952380952384, + "pass@2": 0.4771494627624668, + "pass@4": 0.5758954558693551, + "pass@8": 0.6572878262769214, + "pass@16": 0.7219972024993643, + "pass@32": 0.7752524575772055, + "pass@64": 0.8196462374053157, + "pass@128": 0.8541666666666666 + }, + "20": { + "pass@1": 0.3302067739520958, + "pass@2": 0.4390511545806021, + "pass@4": 0.5388564448717861, + "pass@8": 0.6219737802822332, + "pass@16": 0.6834840446587457, + "pass@32": 0.7260101372704013, + "pass@64": 0.7543047807447868, + "pass@128": 0.7724550898203593 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 662886, + "answer_accuracy": 0.827416020929861, + "avg_response_len": 307.85501627656174, + "resp_tokens_sum": 246638662, + "pass_at_k": { + "pass@1": 0.827416020929861, + "pass@2": 0.8638326950922894, + "pass@4": 0.8917031383918479, + "pass@8": 0.9122528988397226, + "pass@16": 0.9274143119764798, + "pass@32": 0.9392260387639582, + "pass@64": 0.9489087963224284, + "pass@128": 0.9565425786866911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9665578358208955, + "pass@2": 0.9753995769185567, + "pass@4": 0.9812427446569179, + "pass@8": 0.984445931787918, + "pass@16": 0.9862398280511495, + "pass@32": 0.9878623718754762, + "pass@64": 0.9903043768983087, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999964130759167, + "pass@4": 0.9999999986335528, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9924632352941176, + "pass@2": 0.9969336208893007, + "pass@4": 0.9993456238007014, + "pass@8": 0.9999676306603055, + "pass@16": 0.9999999404169826, + "pass@32": 0.9999999999999771, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9984966482649842, + "pass@2": 0.9993359404108397, + "pass@4": 0.9998654172565968, + "pass@8": 0.9999951072498166, + "pass@16": 0.9999999962922006, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966871044303798, + "pass@2": 0.9985314088507925, + "pass@4": 0.9996055398138525, + "pass@8": 0.9999623956767472, + "pass@16": 0.999999710126808, + "pass@32": 0.9999999999966581, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9901229354354354, + "pass@2": 0.9935347503014826, + "pass@4": 0.9964370444685404, + "pass@8": 0.9986250019627753, + "pass@16": 0.9996774337286254, + "pass@32": 0.9999786753419787, + "pass@64": 0.9999999842832614, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965082395498392, + "pass@2": 0.9982063511152746, + "pass@4": 0.9990804087591943, + "pass@8": 0.9997496498520251, + "pass@16": 0.9999841910388799, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9600446428571429, + "pass@2": 0.9693546119235098, + "pass@4": 0.9754381020824777, + "pass@8": 0.9792596757858676, + "pass@16": 0.9818508644575293, + "pass@32": 0.9845842545836299, + "pass@64": 0.9876931724947077, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9324827981651376, + "pass@2": 0.9487191932143805, + "pass@4": 0.9579715631876291, + "pass@8": 0.964293747505247, + "pass@16": 0.9702628430959813, + "pass@32": 0.9772449442356358, + "pass@64": 0.9851239703402783, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8404224537037037, + "pass@2": 0.8757765413871874, + "pass@4": 0.8961142932943567, + "pass@8": 0.908989364931657, + "pass@16": 0.9210844814038844, + "pass@32": 0.9347247035643859, + "pass@64": 0.9469491976683431, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8556194362017804, + "pass@2": 0.8901566041262641, + "pass@4": 0.9098709404647859, + "pass@8": 0.9239797308298969, + "pass@16": 0.9360531611369882, + "pass@32": 0.9474375907664061, + "pass@64": 0.957745729612481, + "pass@128": 0.9673590504451038 + }, + "15": { + "pass@1": 0.7978877314814815, + "pass@2": 0.8533278500777681, + "pass@4": 0.8859646846806184, + "pass@8": 0.9049570675533932, + "pass@16": 0.9179703098545164, + "pass@32": 0.9292966068716593, + "pass@64": 0.939598699165408, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6572409660766961, + "pass@2": 0.7479694359278098, + "pass@4": 0.8105195647557326, + "pass@8": 0.8520118529571892, + "pass@16": 0.879123562506178, + "pass@32": 0.8977227947218035, + "pass@64": 0.9120114491385409, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5544992469879518, + "pass@2": 0.6537838114268096, + "pass@4": 0.7294969114365216, + "pass@8": 0.7825533883751986, + "pass@16": 0.8195442961880983, + "pass@32": 0.8473326604424434, + "pass@64": 0.871423376114652, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.5292338709677419, + "pass@2": 0.6312952438404874, + "pass@4": 0.7058185550394909, + "pass@8": 0.7574372911381855, + "pass@16": 0.7936015396857868, + "pass@32": 0.8200462776298317, + "pass@64": 0.8399903201774559, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.3309337235649547, + "pass@2": 0.4468046750481719, + "pass@4": 0.5559979029245206, + "pass@8": 0.6488701249194181, + "pass@16": 0.7225968808340559, + "pass@32": 0.7785636517822091, + "pass@64": 0.8182159590626885, + "pass@128": 0.8459214501510574 + }, + "20": { + "pass@1": 0.31386073573573575, + "pass@2": 0.4276183171360337, + "pass@4": 0.5344682199071957, + "pass@8": 0.6229720662431183, + "pass@16": 0.6884024081151023, + "pass@32": 0.7363216124083445, + "pass@64": 0.776265928622691, + "pass@128": 0.8108108108108109 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..85372b25f23d68e5bc5eaac930f4ef804aac7ebb --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19a7b70400947e5e88ca60d31ae3c67cfed915ec64790bfdeb7df16d4dec519e +size 1790026934 diff --git a/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_metrics.json b/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5b5aca1af3eca50d9f25aafe5eb71164347cd14f --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op17-20", + "id": { + "answer_accuracy": 0.9928307291666667, + "avg_response_len": 199.24223958333334, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9928307291666667, + "pass@2": 0.9950613790463679, + "pass@4": 0.996422512540099, + "pass@8": 0.9973608871376792, + "pass@16": 0.9980819998075972, + "pass@32": 0.9986988415659556, + "pass@64": 0.9991910668259056, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954875, + "pass@2": 0.9656734744094486, + "pass@4": 0.9738502501874767, + "pass@8": 0.980187661645057, + "pass@16": 0.985422074320855, + "pass@32": 0.9899195932558139, + "pass@64": 0.9932405463259857, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998828125, + "pass@2": 0.9999931102362204, + "pass@4": 0.9999999690663667, + "pass@8": 0.9999999999998846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997421875, + "pass@2": 0.9999464812992126, + "pass@4": 0.9999974311023622, + "pass@8": 0.9999999959061942, + "pass@16": 0.9999999999999984, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996453125, + "pass@2": 0.9986684301181104, + "pass@4": 0.9995166817585303, + "pass@8": 0.9998539345097475, + "pass@16": 0.9999828554611712, + "pass@32": 0.9999998583460734, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985390625, + "pass@2": 0.9994984005905512, + "pass@4": 0.999916720472441, + "pass@8": 0.9999972141757824, + "pass@16": 0.9999999979510268, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9969921875, + "pass@2": 0.9979735482283465, + "pass@4": 0.998548175384327, + "pass@8": 0.9989748614222378, + "pass@16": 0.9992987726757218, + "pass@32": 0.9995808364436816, + "pass@64": 0.9998779527452687, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993515625, + "pass@2": 0.9965835383858268, + "pass@4": 0.9981302094113236, + "pass@8": 0.9990134621448256, + "pass@16": 0.9994716291486171, + "pass@32": 0.9997688599880662, + "pass@64": 0.9999711811022796, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954765625, + "pass@2": 0.9972154281496063, + "pass@4": 0.9978431754780654, + "pass@8": 0.998220854435389, + "pass@16": 0.9985626687109834, + "pass@32": 0.9990204260599655, + "pass@64": 0.9996299212598424, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.954875, + "2": 1.0, + "3": 0.9998828125, + "4": 0.9997421875, + "5": 0.996453125, + "6": 0.9985390625, + "7": 0.9969921875, + "8": 0.993515625, + "9": 0.9954765625 + }, + "per_op_avg_response_len": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376784, + "answer_accuracy": 0.9941320499831138, + "avg_response_len": 189.61737483113814, + "resp_tokens_sum": 71866502, + "pass_at_k": { + "pass@1": 0.9941320499831138, + "pass@2": 0.9960113031083881, + "pass@4": 0.996948029229376, + "pass@8": 0.9975861451314317, + "pass@16": 0.9981962235801742, + "pass@32": 0.9987960607409337, + "pass@64": 0.9992668397522212, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9613753687315634, + "pass@2": 0.970923193505679, + "pass@4": 0.97719835463045, + "pass@8": 0.9817599651022448, + "pass@16": 0.9862649009949807, + "pass@32": 0.9907192299119268, + "pass@64": 0.9939562234532361, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99921875, + "pass@2": 0.9998327540600395, + "pass@4": 0.9999919721948819, + "pass@8": 0.9999999872068569, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9957093253968254, + "pass@2": 0.9977174728158981, + "pass@4": 0.9987727721534809, + "pass@8": 0.9995512405280322, + "pass@16": 0.9999456111182624, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9998802837238909, + "pass@4": 0.999995804229561, + "pass@8": 0.9999999967283617, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984051405325444, + "pass@2": 0.999718993151004, + "pass@4": 0.9999875685842522, + "pass@8": 0.9999999772191345, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976712740384616, + "pass@2": 0.9997326430950938, + "pass@4": 0.9999875658571523, + "pass@8": 0.9999999688739251, + "pass@16": 0.9999999999999599, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972888017078853, + "pass@4": 0.997440846918783, + "pass@8": 0.997682618016843, + "pass@16": 0.9981192931869433, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387347, + "answer_accuracy": 0.991529632208388, + "avg_response_len": 197.23375808895807, + "resp_tokens_sum": 77050551, + "pass_at_k": { + "pass@1": 0.991529632208388, + "pass@2": 0.9937921499778128, + "pass@4": 0.9952555532561216, + "pass@8": 0.9962415224550635, + "pass@16": 0.9970253819958741, + "pass@32": 0.9978481282955417, + "pass@64": 0.9986697318155959, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.939704754601227, + "pass@2": 0.951967522704217, + "pass@4": 0.9620432760322137, + "pass@8": 0.9700759835321355, + "pass@16": 0.9765457934086171, + "pass@32": 0.9828590570934914, + "pass@64": 0.9886812890839152, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984148550724637, + "pass@2": 0.9996868937578454, + "pass@4": 0.9999772702325253, + "pass@8": 0.9999998752549307, + "pass@16": 0.999999999998829, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9977390759312321, + "pass@2": 0.9992202129819733, + "pass@4": 0.9998640429616784, + "pass@8": 0.9999948677008761, + "pass@16": 0.9999999954075054, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9953937861271677, + "pass@2": 0.9963599033953849, + "pass@4": 0.9971111784798576, + "pass@8": 0.9976008320837435, + "pass@16": 0.9980702739375236, + "pass@32": 0.9987903426808435, + "pass@64": 0.999647262300305, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956646126760563, + "pass@2": 0.9985024814239768, + "pass@4": 0.999711590540619, + "pass@8": 0.9999853934625496, + "pass@16": 0.9999999656383994, + "pass@32": 0.9999999999999744, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9948540419161677, + "pass@2": 0.9966571532132587, + "pass@4": 0.9971588291358788, + "pass@8": 0.9973697231456297, + "pass@16": 0.9977102880819406, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379610, + "answer_accuracy": 0.992870145631068, + "avg_response_len": 210.83551378891866, + "resp_tokens_sum": 80610007, + "pass_at_k": { + "pass@1": 0.992870145631068, + "pass@2": 0.9954165722065962, + "pass@4": 0.9970939235939164, + "pass@8": 0.9982813130137579, + "pass@16": 0.9990483811168637, + "pass@32": 0.9994716942355882, + "pass@64": 0.999648633219158, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630597014925373, + "pass@2": 0.9736988042073099, + "pass@4": 0.981951940242544, + "pass@8": 0.9884366055042368, + "pass@16": 0.9932069977085, + "pass@32": 0.995981258815499, + "pass@64": 0.9969530935342759, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997266763848397, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951516544117647, + "pass@2": 0.9985160230430754, + "pass@4": 0.9997385302940074, + "pass@8": 0.9999862846484304, + "pass@16": 0.9999999645862426, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9994023075585583, + "pass@4": 0.9998913907449267, + "pass@8": 0.9999968657441765, + "pass@16": 0.9999999985924529, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972310126582279, + "pass@2": 0.9978734239509618, + "pass@4": 0.9985819919266421, + "pass@8": 0.9993828519658078, + "pass@16": 0.999893854092864, + "pass@32": 0.9999980312535122, + "pass@64": 0.999999999966339, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.987331081081081, + "pass@2": 0.9915873058097466, + "pass@4": 0.9947041267927096, + "pass@8": 0.997053024555423, + "pass@16": 0.9984133373783722, + "pass@32": 0.999305885850076, + "pass@64": 0.9999134567636024, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9977312359166519, + "pass@4": 0.9990373822846098, + "pass@8": 0.9997493167484548, + "pass@16": 0.9999841910297439, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.73775, + "avg_response_len": 375.910928125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.73775, + "pass@2": 0.794706053149587, + "pass@4": 0.8341085378327668, + "pass@8": 0.8628744719585935, + "pass@16": 0.8848724928697924, + "pass@32": 0.9026449216928464, + "pass@64": 0.9174809125133008, + "pass@128": 0.93 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9524765625, + "pass@2": 0.9629094488188978, + "pass@4": 0.9701072982752155, + "pass@8": 0.9754230524252512, + "pass@16": 0.9793903666819255, + "pass@32": 0.9827255399046864, + "pass@64": 0.9855884699571937, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9019453125, + "pass@2": 0.922604576771654, + "pass@4": 0.9358301445444318, + "pass@8": 0.9450701800335944, + "pass@16": 0.9525441316300564, + "pass@32": 0.9599660340270201, + "pass@64": 0.9672012514625326, + "pass@128": 0.973 + }, + "13": { + "pass@1": 0.8347734375, + "pass@2": 0.8710346948818899, + "pass@4": 0.8919833366141726, + "pass@8": 0.9061311839731385, + "pass@16": 0.9180586164072557, + "pass@32": 0.9293484836585746, + "pass@64": 0.9397442533843364, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.861046875, + "pass@2": 0.8973094242125992, + "pass@4": 0.9173674203224592, + "pass@8": 0.9306128578750045, + "pass@16": 0.9412425628429993, + "pass@32": 0.9508925861196847, + "pass@64": 0.9593212166041116, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8366875, + "pass@2": 0.8811536663385828, + "pass@4": 0.9086343062429686, + "pass@8": 0.9272380159376128, + "pass@16": 0.9403654342189126, + "pass@32": 0.9496850768606452, + "pass@64": 0.956857592627593, + "pass@128": 0.963 + }, + "16": { + "pass@1": 0.7048203125, + "pass@2": 0.7783625738188973, + "pass@4": 0.8265161660104984, + "pass@8": 0.8593674600542125, + "pass@16": 0.8829819020113596, + "pass@32": 0.9010030449875494, + "pass@64": 0.9160504722934611, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.6717890625, + "pass@2": 0.7463127460629921, + "pass@4": 0.7963798781402323, + "pass@8": 0.8328647227302957, + "pass@16": 0.8607759566392034, + "pass@32": 0.8828159686152725, + "pass@64": 0.8998461134101179, + "pass@128": 0.912 + }, + "18": { + "pass@1": 0.645828125, + "pass@2": 0.7231929133858277, + "pass@4": 0.7766033110236222, + "pass@8": 0.8153549865324885, + "pass@16": 0.8446173110762112, + "pass@32": 0.8681309150288821, + "pass@64": 0.8872269659541829, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.473640625, + "pass@2": 0.5737501230314962, + "pass@4": 0.6525852853393315, + "pass@8": 0.7137386336937745, + "pass@16": 0.7614976906757347, + "pass@32": 0.7998671857278985, + "pass@64": 0.8322612553363816, + "pass@128": 0.86 + }, + "20": { + "pass@1": 0.4944921875, + "pass@2": 0.5904303641732278, + "pass@4": 0.6650782318147722, + "pass@8": 0.7229436263306257, + "pass@16": 0.7672509565141948, + "pass@32": 0.8020143819982127, + "pass@64": 0.8307115341030594, + "pass@128": 0.856 + } + }, + "per_op_accuracy": { + "11": 0.9524765625, + "12": 0.9019453125, + "13": 0.8347734375, + "14": 0.861046875, + "15": 0.8366875, + "16": 0.7048203125, + "17": 0.6717890625, + "18": 0.645828125, + "19": 0.473640625, + "20": 0.4944921875 + }, + "per_op_avg_response_len": { + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 316722, + "answer_accuracy": 0.7353315378900446, + "avg_response_len": 357.8785661218425, + "resp_tokens_sum": 154145456, + "pass_at_k": { + "pass@1": 0.7353315378900446, + "pass@2": 0.791600330813965, + "pass@4": 0.8304551045996692, + "pass@8": 0.8583848958316062, + "pass@16": 0.8795357228155043, + "pass@32": 0.8969488755607409, + "pass@64": 0.91220347434442, + "pass@128": 0.9254086181277861 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9425017705382436, + "pass@2": 0.9548415159153264, + "pass@4": 0.9637315494911575, + "pass@8": 0.9707074281466466, + "pass@16": 0.9764958731179654, + "pass@32": 0.9818812626501979, + "pass@64": 0.9869854923560497, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.884622175141243, + "pass@2": 0.9070330224876552, + "pass@4": 0.9212047249390439, + "pass@8": 0.9304309349365792, + "pass@16": 0.93734400736929, + "pass@32": 0.9441033505020082, + "pass@64": 0.9506885517975576, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8826244816478657, + "pass@4": 0.9036141996748837, + "pass@8": 0.9170823611637505, + "pass@16": 0.9287953901231346, + "pass@32": 0.9401314416183841, + "pass@64": 0.9503814510788167, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8548481308411215, + "pass@2": 0.890110644025805, + "pass@4": 0.9090667565502909, + "pass@8": 0.9206775494670285, + "pass@16": 0.929940970807209, + "pass@32": 0.9389608922498822, + "pass@64": 0.9479723802614771, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8491239161849711, + "pass@2": 0.891381323676665, + "pass@4": 0.9182221973878987, + "pass@8": 0.9374987178643029, + "pass@16": 0.9500343576067202, + "pass@32": 0.9582461300623976, + "pass@64": 0.9655452112581264, + "pass@128": 0.9710982658959537 + }, + "16": { + "pass@1": 0.6878780241935484, + "pass@2": 0.7575481807213613, + "pass@4": 0.8034270958065721, + "pass@8": 0.8332422545728394, + "pass@16": 0.8555616965823695, + "pass@32": 0.8752240577779585, + "pass@64": 0.8934588712339225, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.6582821531791907, + "pass@2": 0.7333793693960219, + "pass@4": 0.7873619814864179, + "pass@8": 0.8268060950621734, + "pass@16": 0.8541651488113018, + "pass@32": 0.8744259072106768, + "pass@64": 0.8921529102764344, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.6214508928571428, + "pass@2": 0.702418799212598, + "pass@4": 0.7587747707429429, + "pass@8": 0.7988930690855568, + "pass@16": 0.8295439274930722, + "pass@32": 0.8551884496518796, + "pass@64": 0.8759012824017393, + "pass@128": 0.8942857142857142 + }, + "19": { + "pass@1": 0.4751783033033033, + "pass@2": 0.5727038554302333, + "pass@4": 0.6456607301902577, + "pass@8": 0.7009178523026034, + "pass@16": 0.7443995896195046, + "pass@32": 0.780325551418712, + "pass@64": 0.8128709427686198, + "pass@128": 0.8408408408408409 + }, + "20": { + "pass@1": 0.5220298423423423, + "pass@2": 0.6148334604289327, + "pass@4": 0.6857166376162438, + "pass@8": 0.7404605152943717, + "pass@16": 0.782629162910974, + "pass@32": 0.8153740821980516, + "pass@64": 0.8413122377082638, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 321398, + "answer_accuracy": 0.7466315417781743, + "avg_response_len": 371.1641716845079, + "resp_tokens_sum": 159772814, + "pass_at_k": { + "pass@1": 0.7466315417781743, + "pass@2": 0.8018654033238107, + "pass@4": 0.8403893998921449, + "pass@8": 0.8696697999346771, + "pass@16": 0.8927550391117525, + "pass@32": 0.9113634762036628, + "pass@64": 0.9265986541078121, + "pass@128": 0.9396372286648825 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.961902296686747, + "pass@2": 0.9702211898776207, + "pass@4": 0.9745907889149398, + "pass@8": 0.9773632430076152, + "pass@16": 0.9793331174520167, + "pass@32": 0.9809097132926594, + "pass@64": 0.9826862351534986, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.898951802507837, + "pass@2": 0.9214403401377336, + "pass@4": 0.9366077708154816, + "pass@8": 0.9473168294237594, + "pass@16": 0.9557104493167832, + "pass@32": 0.9632188528557657, + "pass@64": 0.9697692688760374, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.8447785364145658, + "pass@2": 0.8756985569818482, + "pass@4": 0.8945737355309575, + "pass@8": 0.9089284772307276, + "pass@16": 0.9205185085929912, + "pass@32": 0.9308260468285129, + "pass@64": 0.9408199495389861, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8758223684210527, + "pass@2": 0.9139463035870513, + "pass@4": 0.9350603600536769, + "pass@8": 0.9487797027077906, + "pass@16": 0.9591678574023239, + "pass@32": 0.9683737126824168, + "pass@64": 0.975387169545268, + "pass@128": 0.97953216374269 + }, + "15": { + "pass@1": 0.8341619318181818, + "pass@2": 0.8827543396564063, + "pass@4": 0.9144763560804903, + "pass@8": 0.9369795398942257, + "pass@16": 0.9540667236743633, + "pass@32": 0.9664573273663126, + "pass@64": 0.9747860909253457, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.7109375, + "pass@2": 0.7824315930861211, + "pass@4": 0.8306402492957612, + "pass@8": 0.8662793559814501, + "pass@16": 0.8918811624991876, + "pass@32": 0.9103467438874276, + "pass@64": 0.9247234008320129, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.710549301242236, + "pass@2": 0.7828643107790872, + "pass@4": 0.8275231751892806, + "pass@8": 0.8615275014644767, + "pass@16": 0.8904357410433366, + "pass@32": 0.9134983203987874, + "pass@64": 0.9284336977111665, + "pass@128": 0.937888198757764 + }, + "18": { + "pass@1": 0.6790900735294118, + "pass@2": 0.7502435299907362, + "pass@4": 0.800968797834094, + "pass@8": 0.8404703821156387, + "pass@16": 0.8728190470546322, + "pass@32": 0.8996975551996976, + "pass@64": 0.9220785178168743, + "pass@128": 0.9411764705882353 + }, + "19": { + "pass@1": 0.4720982142857143, + "pass@2": 0.565974175103112, + "pass@4": 0.6416543777451924, + "pass@8": 0.7033531553412125, + "pass@16": 0.7537467764321175, + "pass@32": 0.7949539015604953, + "pass@64": 0.831013697436013, + "pass@128": 0.8660714285714286 + }, + "20": { + "pass@1": 0.4807260479041916, + "pass@2": 0.5753041161770944, + "pass@4": 0.6497437565813254, + "pass@8": 0.7074401183175504, + "pass@16": 0.7516001349159216, + "pass@32": 0.787087003435915, + "pass@64": 0.8178073059749411, + "pass@128": 0.8473053892215568 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 306200, + "answer_accuracy": 0.7311086491442543, + "avg_response_len": 399.3345956219438, + "resp_tokens_sum": 167247718, + "pass_at_k": { + "pass@1": 0.7311086491442543, + "pass@2": 0.7905415852472955, + "pass@4": 0.8314102687996849, + "pass@8": 0.8605073373876343, + "pass@16": 0.8822591763114017, + "pass@32": 0.8995418338000026, + "pass@64": 0.9135370294007938, + "pass@128": 0.9248166259168704 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9537202380952381, + "pass@2": 0.9642443132108488, + "pass@4": 0.972526728206593, + "pass@8": 0.9786626463839903, + "pass@16": 0.9826943761181406, + "pass@32": 0.9855854900825515, + "pass@64": 0.9870817812207828, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.9236190749235474, + "pass@2": 0.9405976095017944, + "pass@4": 0.9509045658283538, + "pass@8": 0.9587264846479097, + "pass@16": 0.9659104577653655, + "pass@32": 0.9739652412486856, + "pass@64": 0.9825721937453857, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.8131028163580247, + "pass@2": 0.8544848777583361, + "pass@4": 0.877677726337449, + "pass@8": 0.8922668036744839, + "pass@16": 0.9047770968835759, + "pass@32": 0.9171038736559595, + "pass@64": 0.9280859521752334, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8519566023738873, + "pass@2": 0.8872827052033923, + "pass@4": 0.9073185410429038, + "pass@8": 0.9216400776558572, + "pass@16": 0.9338163322916623, + "pass@32": 0.9445172996144986, + "pass@64": 0.9538269155955367, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.8259789737654321, + "pass@2": 0.8686012230242051, + "pass@4": 0.8924451495067742, + "pass@8": 0.9063586771341635, + "pass@16": 0.9160849619581726, + "pass@32": 0.9234598698400384, + "pass@64": 0.9293195664411008, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.7139795353982301, + "pass@2": 0.7931833292685756, + "pass@4": 0.8433599964827404, + "pass@8": 0.8761012011420178, + "pass@16": 0.8988421475917746, + "pass@32": 0.9149023008020526, + "pass@64": 0.9277295236545959, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.6482727786144579, + "pass@2": 0.7243408921117538, + "pass@4": 0.7755728016113949, + "pass@8": 0.8113793926723576, + "pass@16": 0.8388991160678905, + "pass@32": 0.8618016432288235, + "pass@64": 0.880137216239385, + "pass@128": 0.891566265060241 + }, + "18": { + "pass@1": 0.6368699596774193, + "pass@2": 0.7169791402082805, + "pass@4": 0.7700088709677413, + "pass@8": 0.8063951046233131, + "pass@16": 0.8307047111453599, + "pass@32": 0.848121899622024, + "pass@64": 0.8617897453414103, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.47365936555891236, + "pass@2": 0.5826961218688298, + "pass@4": 0.6706477077449906, + "pass@8": 0.7371792407322068, + "pass@16": 0.7865671010611726, + "pass@32": 0.8245143993387958, + "pass@64": 0.8530351330994856, + "pass@128": 0.8731117824773413 + }, + "20": { + "pass@1": 0.480762012012012, + "pass@2": 0.5811989400818144, + "pass@4": 0.659820350721925, + "pass@8": 0.7209768024610734, + "pass@16": 0.7675705711799523, + "pass@32": 0.8036268873233177, + "pass@64": 0.8330538100617901, + "pass@128": 0.8588588588588588 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8585777138157895, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8585777138157895, + "pass@2": 0.8896112075217446, + "pass@4": 0.9109941047993847, + "pass@8": 0.9265785633592267, + "pass@16": 0.9384980487876894, + "pass@32": 0.9481441468959069, + "pass@64": 0.9561857224508497, + "pass@128": 0.9629473684210527 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954875, + "pass@2": 0.9656734744094486, + "pass@4": 0.9738502501874767, + "pass@8": 0.980187661645057, + "pass@16": 0.985422074320855, + "pass@32": 0.9899195932558139, + "pass@64": 0.9932405463259857, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998828125, + "pass@2": 0.9999931102362204, + "pass@4": 0.9999999690663667, + "pass@8": 0.9999999999998846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997421875, + "pass@2": 0.9999464812992126, + "pass@4": 0.9999974311023622, + "pass@8": 0.9999999959061942, + "pass@16": 0.9999999999999984, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996453125, + "pass@2": 0.9986684301181104, + "pass@4": 0.9995166817585303, + "pass@8": 0.9998539345097475, + "pass@16": 0.9999828554611712, + "pass@32": 0.9999998583460734, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985390625, + "pass@2": 0.9994984005905512, + "pass@4": 0.999916720472441, + "pass@8": 0.9999972141757824, + "pass@16": 0.9999999979510268, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9969921875, + "pass@2": 0.9979735482283465, + "pass@4": 0.998548175384327, + "pass@8": 0.9989748614222378, + "pass@16": 0.9992987726757218, + "pass@32": 0.9995808364436816, + "pass@64": 0.9998779527452687, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993515625, + "pass@2": 0.9965835383858268, + "pass@4": 0.9981302094113236, + "pass@8": 0.9990134621448256, + "pass@16": 0.9994716291486171, + "pass@32": 0.9997688599880662, + "pass@64": 0.9999711811022796, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954765625, + "pass@2": 0.9972154281496063, + "pass@4": 0.9978431754780654, + "pass@8": 0.998220854435389, + "pass@16": 0.9985626687109834, + "pass@32": 0.9990204260599655, + "pass@64": 0.9996299212598424, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9524765625, + "pass@2": 0.9629094488188978, + "pass@4": 0.9701072982752155, + "pass@8": 0.9754230524252512, + "pass@16": 0.9793903666819255, + "pass@32": 0.9827255399046864, + "pass@64": 0.9855884699571937, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9019453125, + "pass@2": 0.922604576771654, + "pass@4": 0.9358301445444318, + "pass@8": 0.9450701800335944, + "pass@16": 0.9525441316300564, + "pass@32": 0.9599660340270201, + "pass@64": 0.9672012514625326, + "pass@128": 0.973 + }, + "13": { + "pass@1": 0.8347734375, + "pass@2": 0.8710346948818899, + "pass@4": 0.8919833366141726, + "pass@8": 0.9061311839731385, + "pass@16": 0.9180586164072557, + "pass@32": 0.9293484836585746, + "pass@64": 0.9397442533843364, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.861046875, + "pass@2": 0.8973094242125992, + "pass@4": 0.9173674203224592, + "pass@8": 0.9306128578750045, + "pass@16": 0.9412425628429993, + "pass@32": 0.9508925861196847, + "pass@64": 0.9593212166041116, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8366875, + "pass@2": 0.8811536663385828, + "pass@4": 0.9086343062429686, + "pass@8": 0.9272380159376128, + "pass@16": 0.9403654342189126, + "pass@32": 0.9496850768606452, + "pass@64": 0.956857592627593, + "pass@128": 0.963 + }, + "16": { + "pass@1": 0.7048203125, + "pass@2": 0.7783625738188973, + "pass@4": 0.8265161660104984, + "pass@8": 0.8593674600542125, + "pass@16": 0.8829819020113596, + "pass@32": 0.9010030449875494, + "pass@64": 0.9160504722934611, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.6717890625, + "pass@2": 0.7463127460629921, + "pass@4": 0.7963798781402323, + "pass@8": 0.8328647227302957, + "pass@16": 0.8607759566392034, + "pass@32": 0.8828159686152725, + "pass@64": 0.8998461134101179, + "pass@128": 0.912 + }, + "18": { + "pass@1": 0.645828125, + "pass@2": 0.7231929133858277, + "pass@4": 0.7766033110236222, + "pass@8": 0.8153549865324885, + "pass@16": 0.8446173110762112, + "pass@32": 0.8681309150288821, + "pass@64": 0.8872269659541829, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.473640625, + "pass@2": 0.5737501230314962, + "pass@4": 0.6525852853393315, + "pass@8": 0.7137386336937745, + "pass@16": 0.7614976906757347, + "pass@32": 0.7998671857278985, + "pass@64": 0.8322612553363816, + "pass@128": 0.86 + }, + "20": { + "pass@1": 0.4944921875, + "pass@2": 0.5904303641732278, + "pass@4": 0.6650782318147722, + "pass@8": 0.7229436263306257, + "pass@16": 0.7672509565141948, + "pass@32": 0.8020143819982127, + "pass@64": 0.8307115341030594, + "pass@128": 0.856 + } + }, + "per_op_accuracy": { + "10": 0.954875, + "2": 1.0, + "3": 0.9998828125, + "4": 0.9997421875, + "5": 0.996453125, + "6": 0.9985390625, + "7": 0.9969921875, + "8": 0.993515625, + "9": 0.9954765625, + "11": 0.9524765625, + "12": 0.9019453125, + "13": 0.8347734375, + "14": 0.861046875, + "15": 0.8366875, + "16": 0.7048203125, + "17": 0.6717890625, + "18": 0.645828125, + "19": 0.473640625, + "20": 0.4944921875 + }, + "per_op_avg_response_len": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625, + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625, + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 693506, + "answer_accuracy": 0.8564678509326589, + "avg_response_len": 279.12083810859946, + "resp_tokens_sum": 226011958, + "pass_at_k": { + "pass@1": 0.8564678509326589, + "pass@2": 0.8872786249909652, + "pass@4": 0.9083851630613429, + "pass@8": 0.92354058650134, + "pass@16": 0.9350769404513234, + "pass@32": 0.9446203133284556, + "pass@64": 0.9529550748775423, + "pass@128": 0.9601644008852356 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9613753687315634, + "pass@2": 0.970923193505679, + "pass@4": 0.97719835463045, + "pass@8": 0.9817599651022448, + "pass@16": 0.9862649009949807, + "pass@32": 0.9907192299119268, + "pass@64": 0.9939562234532361, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99921875, + "pass@2": 0.9998327540600395, + "pass@4": 0.9999919721948819, + "pass@8": 0.9999999872068569, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9957093253968254, + "pass@2": 0.9977174728158981, + "pass@4": 0.9987727721534809, + "pass@8": 0.9995512405280322, + "pass@16": 0.9999456111182624, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9998802837238909, + "pass@4": 0.999995804229561, + "pass@8": 0.9999999967283617, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984051405325444, + "pass@2": 0.999718993151004, + "pass@4": 0.9999875685842522, + "pass@8": 0.9999999772191345, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976712740384616, + "pass@2": 0.9997326430950938, + "pass@4": 0.9999875658571523, + "pass@8": 0.9999999688739251, + "pass@16": 0.9999999999999599, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972888017078853, + "pass@4": 0.997440846918783, + "pass@8": 0.997682618016843, + "pass@16": 0.9981192931869433, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9425017705382436, + "pass@2": 0.9548415159153264, + "pass@4": 0.9637315494911575, + "pass@8": 0.9707074281466466, + "pass@16": 0.9764958731179654, + "pass@32": 0.9818812626501979, + "pass@64": 0.9869854923560497, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.884622175141243, + "pass@2": 0.9070330224876552, + "pass@4": 0.9212047249390439, + "pass@8": 0.9304309349365792, + "pass@16": 0.93734400736929, + "pass@32": 0.9441033505020082, + "pass@64": 0.9506885517975576, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8826244816478657, + "pass@4": 0.9036141996748837, + "pass@8": 0.9170823611637505, + "pass@16": 0.9287953901231346, + "pass@32": 0.9401314416183841, + "pass@64": 0.9503814510788167, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8548481308411215, + "pass@2": 0.890110644025805, + "pass@4": 0.9090667565502909, + "pass@8": 0.9206775494670285, + "pass@16": 0.929940970807209, + "pass@32": 0.9389608922498822, + "pass@64": 0.9479723802614771, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8491239161849711, + "pass@2": 0.891381323676665, + "pass@4": 0.9182221973878987, + "pass@8": 0.9374987178643029, + "pass@16": 0.9500343576067202, + "pass@32": 0.9582461300623976, + "pass@64": 0.9655452112581264, + "pass@128": 0.9710982658959537 + }, + "16": { + "pass@1": 0.6878780241935484, + "pass@2": 0.7575481807213613, + "pass@4": 0.8034270958065721, + "pass@8": 0.8332422545728394, + "pass@16": 0.8555616965823695, + "pass@32": 0.8752240577779585, + "pass@64": 0.8934588712339225, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.6582821531791907, + "pass@2": 0.7333793693960219, + "pass@4": 0.7873619814864179, + "pass@8": 0.8268060950621734, + "pass@16": 0.8541651488113018, + "pass@32": 0.8744259072106768, + "pass@64": 0.8921529102764344, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.6214508928571428, + "pass@2": 0.702418799212598, + "pass@4": 0.7587747707429429, + "pass@8": 0.7988930690855568, + "pass@16": 0.8295439274930722, + "pass@32": 0.8551884496518796, + "pass@64": 0.8759012824017393, + "pass@128": 0.8942857142857142 + }, + "19": { + "pass@1": 0.4751783033033033, + "pass@2": 0.5727038554302333, + "pass@4": 0.6456607301902577, + "pass@8": 0.7009178523026034, + "pass@16": 0.7443995896195046, + "pass@32": 0.780325551418712, + "pass@64": 0.8128709427686198, + "pass@128": 0.8408408408408409 + }, + "20": { + "pass@1": 0.5220298423423423, + "pass@2": 0.6148334604289327, + "pass@4": 0.6857166376162438, + "pass@8": 0.7404605152943717, + "pass@16": 0.782629162910974, + "pass@32": 0.8153740821980516, + "pass@64": 0.8413122377082638, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 708745, + "answer_accuracy": 0.8631442420109119, + "avg_response_len": 288.41504895752144, + "resp_tokens_sum": 236823365, + "pass_at_k": { + "pass@1": 0.8631442420109119, + "pass@2": 0.8931764603445276, + "pass@4": 0.9140685113600835, + "pass@8": 0.929887554748734, + "pass@16": 0.9423626909406467, + "pass@32": 0.9525094088902438, + "pass@64": 0.960887185544163, + "pass@128": 0.9680436477007015 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.939704754601227, + "pass@2": 0.951967522704217, + "pass@4": 0.9620432760322137, + "pass@8": 0.9700759835321355, + "pass@16": 0.9765457934086171, + "pass@32": 0.9828590570934914, + "pass@64": 0.9886812890839152, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984148550724637, + "pass@2": 0.9996868937578454, + "pass@4": 0.9999772702325253, + "pass@8": 0.9999998752549307, + "pass@16": 0.999999999998829, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9977390759312321, + "pass@2": 0.9992202129819733, + "pass@4": 0.9998640429616784, + "pass@8": 0.9999948677008761, + "pass@16": 0.9999999954075054, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9953937861271677, + "pass@2": 0.9963599033953849, + "pass@4": 0.9971111784798576, + "pass@8": 0.9976008320837435, + "pass@16": 0.9980702739375236, + "pass@32": 0.9987903426808435, + "pass@64": 0.999647262300305, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956646126760563, + "pass@2": 0.9985024814239768, + "pass@4": 0.999711590540619, + "pass@8": 0.9999853934625496, + "pass@16": 0.9999999656383994, + "pass@32": 0.9999999999999744, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9948540419161677, + "pass@2": 0.9966571532132587, + "pass@4": 0.9971588291358788, + "pass@8": 0.9973697231456297, + "pass@16": 0.9977102880819406, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.961902296686747, + "pass@2": 0.9702211898776207, + "pass@4": 0.9745907889149398, + "pass@8": 0.9773632430076152, + "pass@16": 0.9793331174520167, + "pass@32": 0.9809097132926594, + "pass@64": 0.9826862351534986, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.898951802507837, + "pass@2": 0.9214403401377336, + "pass@4": 0.9366077708154816, + "pass@8": 0.9473168294237594, + "pass@16": 0.9557104493167832, + "pass@32": 0.9632188528557657, + "pass@64": 0.9697692688760374, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.8447785364145658, + "pass@2": 0.8756985569818482, + "pass@4": 0.8945737355309575, + "pass@8": 0.9089284772307276, + "pass@16": 0.9205185085929912, + "pass@32": 0.9308260468285129, + "pass@64": 0.9408199495389861, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8758223684210527, + "pass@2": 0.9139463035870513, + "pass@4": 0.9350603600536769, + "pass@8": 0.9487797027077906, + "pass@16": 0.9591678574023239, + "pass@32": 0.9683737126824168, + "pass@64": 0.975387169545268, + "pass@128": 0.97953216374269 + }, + "15": { + "pass@1": 0.8341619318181818, + "pass@2": 0.8827543396564063, + "pass@4": 0.9144763560804903, + "pass@8": 0.9369795398942257, + "pass@16": 0.9540667236743633, + "pass@32": 0.9664573273663126, + "pass@64": 0.9747860909253457, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.7109375, + "pass@2": 0.7824315930861211, + "pass@4": 0.8306402492957612, + "pass@8": 0.8662793559814501, + "pass@16": 0.8918811624991876, + "pass@32": 0.9103467438874276, + "pass@64": 0.9247234008320129, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.710549301242236, + "pass@2": 0.7828643107790872, + "pass@4": 0.8275231751892806, + "pass@8": 0.8615275014644767, + "pass@16": 0.8904357410433366, + "pass@32": 0.9134983203987874, + "pass@64": 0.9284336977111665, + "pass@128": 0.937888198757764 + }, + "18": { + "pass@1": 0.6790900735294118, + "pass@2": 0.7502435299907362, + "pass@4": 0.800968797834094, + "pass@8": 0.8404703821156387, + "pass@16": 0.8728190470546322, + "pass@32": 0.8996975551996976, + "pass@64": 0.9220785178168743, + "pass@128": 0.9411764705882353 + }, + "19": { + "pass@1": 0.4720982142857143, + "pass@2": 0.565974175103112, + "pass@4": 0.6416543777451924, + "pass@8": 0.7033531553412125, + "pass@16": 0.7537467764321175, + "pass@32": 0.7949539015604953, + "pass@64": 0.831013697436013, + "pass@128": 0.8660714285714286 + }, + "20": { + "pass@1": 0.4807260479041916, + "pass@2": 0.5753041161770944, + "pass@4": 0.6497437565813254, + "pass@8": 0.7074401183175504, + "pass@16": 0.7516001349159216, + "pass@32": 0.787087003435915, + "pass@64": 0.8178073059749411, + "pass@128": 0.8473053892215568 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 685810, + "answer_accuracy": 0.8560298170634286, + "avg_response_len": 309.37665386842946, + "resp_tokens_sum": 247857725, + "pass_at_k": { + "pass@1": 0.8560298170634286, + "pass@2": 0.8883146458076645, + "pass@4": 0.910479940771302, + "pass@8": 0.9262575954472617, + "pass@16": 0.9379948137541099, + "pass@32": 0.9472316393793455, + "pass@64": 0.9546323098937601, + "pass@128": 0.9605368269691644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630597014925373, + "pass@2": 0.9736988042073099, + "pass@4": 0.981951940242544, + "pass@8": 0.9884366055042368, + "pass@16": 0.9932069977085, + "pass@32": 0.995981258815499, + "pass@64": 0.9969530935342759, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997266763848397, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951516544117647, + "pass@2": 0.9985160230430754, + "pass@4": 0.9997385302940074, + "pass@8": 0.9999862846484304, + "pass@16": 0.9999999645862426, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9994023075585583, + "pass@4": 0.9998913907449267, + "pass@8": 0.9999968657441765, + "pass@16": 0.9999999985924529, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972310126582279, + "pass@2": 0.9978734239509618, + "pass@4": 0.9985819919266421, + "pass@8": 0.9993828519658078, + "pass@16": 0.999893854092864, + "pass@32": 0.9999980312535122, + "pass@64": 0.999999999966339, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.987331081081081, + "pass@2": 0.9915873058097466, + "pass@4": 0.9947041267927096, + "pass@8": 0.997053024555423, + "pass@16": 0.9984133373783722, + "pass@32": 0.999305885850076, + "pass@64": 0.9999134567636024, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9977312359166519, + "pass@4": 0.9990373822846098, + "pass@8": 0.9997493167484548, + "pass@16": 0.9999841910297439, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9537202380952381, + "pass@2": 0.9642443132108488, + "pass@4": 0.972526728206593, + "pass@8": 0.9786626463839903, + "pass@16": 0.9826943761181406, + "pass@32": 0.9855854900825515, + "pass@64": 0.9870817812207828, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.9236190749235474, + "pass@2": 0.9405976095017944, + "pass@4": 0.9509045658283538, + "pass@8": 0.9587264846479097, + "pass@16": 0.9659104577653655, + "pass@32": 0.9739652412486856, + "pass@64": 0.9825721937453857, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.8131028163580247, + "pass@2": 0.8544848777583361, + "pass@4": 0.877677726337449, + "pass@8": 0.8922668036744839, + "pass@16": 0.9047770968835759, + "pass@32": 0.9171038736559595, + "pass@64": 0.9280859521752334, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8519566023738873, + "pass@2": 0.8872827052033923, + "pass@4": 0.9073185410429038, + "pass@8": 0.9216400776558572, + "pass@16": 0.9338163322916623, + "pass@32": 0.9445172996144986, + "pass@64": 0.9538269155955367, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.8259789737654321, + "pass@2": 0.8686012230242051, + "pass@4": 0.8924451495067742, + "pass@8": 0.9063586771341635, + "pass@16": 0.9160849619581726, + "pass@32": 0.9234598698400384, + "pass@64": 0.9293195664411008, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.7139795353982301, + "pass@2": 0.7931833292685756, + "pass@4": 0.8433599964827404, + "pass@8": 0.8761012011420178, + "pass@16": 0.8988421475917746, + "pass@32": 0.9149023008020526, + "pass@64": 0.9277295236545959, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.6482727786144579, + "pass@2": 0.7243408921117538, + "pass@4": 0.7755728016113949, + "pass@8": 0.8113793926723576, + "pass@16": 0.8388991160678905, + "pass@32": 0.8618016432288235, + "pass@64": 0.880137216239385, + "pass@128": 0.891566265060241 + }, + "18": { + "pass@1": 0.6368699596774193, + "pass@2": 0.7169791402082805, + "pass@4": 0.7700088709677413, + "pass@8": 0.8063951046233131, + "pass@16": 0.8307047111453599, + "pass@32": 0.848121899622024, + "pass@64": 0.8617897453414103, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.47365936555891236, + "pass@2": 0.5826961218688298, + "pass@4": 0.6706477077449906, + "pass@8": 0.7371792407322068, + "pass@16": 0.7865671010611726, + "pass@32": 0.8245143993387958, + "pass@64": 0.8530351330994856, + "pass@128": 0.8731117824773413 + }, + "20": { + "pass@1": 0.480762012012012, + "pass@2": 0.5811989400818144, + "pass@4": 0.659820350721925, + "pass@8": 0.7209768024610734, + "pass@16": 0.7675705711799523, + "pass@32": 0.8036268873233177, + "pass@64": 0.8330538100617901, + "pass@128": 0.8588588588588588 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c390e257d32382c78225195821716dec27d56cd1 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfa5336b4cea10555b92514d69fa20bd0ba466d789970501d4175834767b0ed5 +size 2713490348 diff --git a/results/composition/SDAR-100M-block4-epoch4-op17-20/summary.csv b/results/composition/SDAR-100M-block4-epoch4-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8ace68e34ff36b952189619a68672be8bb4e6337 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op17-20,0.9928307291666667,0.73775,0.8585777138157895,0.0,0.0,0.0,199.24223958333334,375.910928125,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch4-op17-20/summary.json b/results/composition/SDAR-100M-block4-epoch4-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ea76301e4e6ec38fed2724c30850d96785e069b9 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op17-20", + "id_acc": 0.9928307291666667, + "ood_acc": 0.73775, + "total_acc": 0.8585777138157895, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.24223958333334, + "ood_avg_resp_len": 375.910928125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 693506, + "answer_accuracy": 0.8564678509326589, + "avg_response_len": 279.12083810859946, + "resp_tokens_sum": 226011958, + "pass_at_k": { + "pass@1": 0.8564678509326589, + "pass@2": 0.8872786249909652, + "pass@4": 0.9083851630613429, + "pass@8": 0.92354058650134, + "pass@16": 0.9350769404513234, + "pass@32": 0.9446203133284556, + "pass@64": 0.9529550748775423, + "pass@128": 0.9601644008852356 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9613753687315634, + "pass@2": 0.970923193505679, + "pass@4": 0.97719835463045, + "pass@8": 0.9817599651022448, + "pass@16": 0.9862649009949807, + "pass@32": 0.9907192299119268, + "pass@64": 0.9939562234532361, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99921875, + "pass@2": 0.9998327540600395, + "pass@4": 0.9999919721948819, + "pass@8": 0.9999999872068569, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9957093253968254, + "pass@2": 0.9977174728158981, + "pass@4": 0.9987727721534809, + "pass@8": 0.9995512405280322, + "pass@16": 0.9999456111182624, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9998802837238909, + "pass@4": 0.999995804229561, + "pass@8": 0.9999999967283617, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984051405325444, + "pass@2": 0.999718993151004, + "pass@4": 0.9999875685842522, + "pass@8": 0.9999999772191345, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976712740384616, + "pass@2": 0.9997326430950938, + "pass@4": 0.9999875658571523, + "pass@8": 0.9999999688739251, + "pass@16": 0.9999999999999599, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972888017078853, + "pass@4": 0.997440846918783, + "pass@8": 0.997682618016843, + "pass@16": 0.9981192931869433, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9425017705382436, + "pass@2": 0.9548415159153264, + "pass@4": 0.9637315494911575, + "pass@8": 0.9707074281466466, + "pass@16": 0.9764958731179654, + "pass@32": 0.9818812626501979, + "pass@64": 0.9869854923560497, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.884622175141243, + "pass@2": 0.9070330224876552, + "pass@4": 0.9212047249390439, + "pass@8": 0.9304309349365792, + "pass@16": 0.93734400736929, + "pass@32": 0.9441033505020082, + "pass@64": 0.9506885517975576, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8826244816478657, + "pass@4": 0.9036141996748837, + "pass@8": 0.9170823611637505, + "pass@16": 0.9287953901231346, + "pass@32": 0.9401314416183841, + "pass@64": 0.9503814510788167, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8548481308411215, + "pass@2": 0.890110644025805, + "pass@4": 0.9090667565502909, + "pass@8": 0.9206775494670285, + "pass@16": 0.929940970807209, + "pass@32": 0.9389608922498822, + "pass@64": 0.9479723802614771, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8491239161849711, + "pass@2": 0.891381323676665, + "pass@4": 0.9182221973878987, + "pass@8": 0.9374987178643029, + "pass@16": 0.9500343576067202, + "pass@32": 0.9582461300623976, + "pass@64": 0.9655452112581264, + "pass@128": 0.9710982658959537 + }, + "16": { + "pass@1": 0.6878780241935484, + "pass@2": 0.7575481807213613, + "pass@4": 0.8034270958065721, + "pass@8": 0.8332422545728394, + "pass@16": 0.8555616965823695, + "pass@32": 0.8752240577779585, + "pass@64": 0.8934588712339225, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.6582821531791907, + "pass@2": 0.7333793693960219, + "pass@4": 0.7873619814864179, + "pass@8": 0.8268060950621734, + "pass@16": 0.8541651488113018, + "pass@32": 0.8744259072106768, + "pass@64": 0.8921529102764344, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.6214508928571428, + "pass@2": 0.702418799212598, + "pass@4": 0.7587747707429429, + "pass@8": 0.7988930690855568, + "pass@16": 0.8295439274930722, + "pass@32": 0.8551884496518796, + "pass@64": 0.8759012824017393, + "pass@128": 0.8942857142857142 + }, + "19": { + "pass@1": 0.4751783033033033, + "pass@2": 0.5727038554302333, + "pass@4": 0.6456607301902577, + "pass@8": 0.7009178523026034, + "pass@16": 0.7443995896195046, + "pass@32": 0.780325551418712, + "pass@64": 0.8128709427686198, + "pass@128": 0.8408408408408409 + }, + "20": { + "pass@1": 0.5220298423423423, + "pass@2": 0.6148334604289327, + "pass@4": 0.6857166376162438, + "pass@8": 0.7404605152943717, + "pass@16": 0.782629162910974, + "pass@32": 0.8153740821980516, + "pass@64": 0.8413122377082638, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 708745, + "answer_accuracy": 0.8631442420109119, + "avg_response_len": 288.41504895752144, + "resp_tokens_sum": 236823365, + "pass_at_k": { + "pass@1": 0.8631442420109119, + "pass@2": 0.8931764603445276, + "pass@4": 0.9140685113600835, + "pass@8": 0.929887554748734, + "pass@16": 0.9423626909406467, + "pass@32": 0.9525094088902438, + "pass@64": 0.960887185544163, + "pass@128": 0.9680436477007015 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.939704754601227, + "pass@2": 0.951967522704217, + "pass@4": 0.9620432760322137, + "pass@8": 0.9700759835321355, + "pass@16": 0.9765457934086171, + "pass@32": 0.9828590570934914, + "pass@64": 0.9886812890839152, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984148550724637, + "pass@2": 0.9996868937578454, + "pass@4": 0.9999772702325253, + "pass@8": 0.9999998752549307, + "pass@16": 0.999999999998829, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9977390759312321, + "pass@2": 0.9992202129819733, + "pass@4": 0.9998640429616784, + "pass@8": 0.9999948677008761, + "pass@16": 0.9999999954075054, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9953937861271677, + "pass@2": 0.9963599033953849, + "pass@4": 0.9971111784798576, + "pass@8": 0.9976008320837435, + "pass@16": 0.9980702739375236, + "pass@32": 0.9987903426808435, + "pass@64": 0.999647262300305, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956646126760563, + "pass@2": 0.9985024814239768, + "pass@4": 0.999711590540619, + "pass@8": 0.9999853934625496, + "pass@16": 0.9999999656383994, + "pass@32": 0.9999999999999744, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9948540419161677, + "pass@2": 0.9966571532132587, + "pass@4": 0.9971588291358788, + "pass@8": 0.9973697231456297, + "pass@16": 0.9977102880819406, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.961902296686747, + "pass@2": 0.9702211898776207, + "pass@4": 0.9745907889149398, + "pass@8": 0.9773632430076152, + "pass@16": 0.9793331174520167, + "pass@32": 0.9809097132926594, + "pass@64": 0.9826862351534986, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.898951802507837, + "pass@2": 0.9214403401377336, + "pass@4": 0.9366077708154816, + "pass@8": 0.9473168294237594, + "pass@16": 0.9557104493167832, + "pass@32": 0.9632188528557657, + "pass@64": 0.9697692688760374, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.8447785364145658, + "pass@2": 0.8756985569818482, + "pass@4": 0.8945737355309575, + "pass@8": 0.9089284772307276, + "pass@16": 0.9205185085929912, + "pass@32": 0.9308260468285129, + "pass@64": 0.9408199495389861, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8758223684210527, + "pass@2": 0.9139463035870513, + "pass@4": 0.9350603600536769, + "pass@8": 0.9487797027077906, + "pass@16": 0.9591678574023239, + "pass@32": 0.9683737126824168, + "pass@64": 0.975387169545268, + "pass@128": 0.97953216374269 + }, + "15": { + "pass@1": 0.8341619318181818, + "pass@2": 0.8827543396564063, + "pass@4": 0.9144763560804903, + "pass@8": 0.9369795398942257, + "pass@16": 0.9540667236743633, + "pass@32": 0.9664573273663126, + "pass@64": 0.9747860909253457, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.7109375, + "pass@2": 0.7824315930861211, + "pass@4": 0.8306402492957612, + "pass@8": 0.8662793559814501, + "pass@16": 0.8918811624991876, + "pass@32": 0.9103467438874276, + "pass@64": 0.9247234008320129, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.710549301242236, + "pass@2": 0.7828643107790872, + "pass@4": 0.8275231751892806, + "pass@8": 0.8615275014644767, + "pass@16": 0.8904357410433366, + "pass@32": 0.9134983203987874, + "pass@64": 0.9284336977111665, + "pass@128": 0.937888198757764 + }, + "18": { + "pass@1": 0.6790900735294118, + "pass@2": 0.7502435299907362, + "pass@4": 0.800968797834094, + "pass@8": 0.8404703821156387, + "pass@16": 0.8728190470546322, + "pass@32": 0.8996975551996976, + "pass@64": 0.9220785178168743, + "pass@128": 0.9411764705882353 + }, + "19": { + "pass@1": 0.4720982142857143, + "pass@2": 0.565974175103112, + "pass@4": 0.6416543777451924, + "pass@8": 0.7033531553412125, + "pass@16": 0.7537467764321175, + "pass@32": 0.7949539015604953, + "pass@64": 0.831013697436013, + "pass@128": 0.8660714285714286 + }, + "20": { + "pass@1": 0.4807260479041916, + "pass@2": 0.5753041161770944, + "pass@4": 0.6497437565813254, + "pass@8": 0.7074401183175504, + "pass@16": 0.7516001349159216, + "pass@32": 0.787087003435915, + "pass@64": 0.8178073059749411, + "pass@128": 0.8473053892215568 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 685810, + "answer_accuracy": 0.8560298170634286, + "avg_response_len": 309.37665386842946, + "resp_tokens_sum": 247857725, + "pass_at_k": { + "pass@1": 0.8560298170634286, + "pass@2": 0.8883146458076645, + "pass@4": 0.910479940771302, + "pass@8": 0.9262575954472617, + "pass@16": 0.9379948137541099, + "pass@32": 0.9472316393793455, + "pass@64": 0.9546323098937601, + "pass@128": 0.9605368269691644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630597014925373, + "pass@2": 0.9736988042073099, + "pass@4": 0.981951940242544, + "pass@8": 0.9884366055042368, + "pass@16": 0.9932069977085, + "pass@32": 0.995981258815499, + "pass@64": 0.9969530935342759, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997266763848397, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951516544117647, + "pass@2": 0.9985160230430754, + "pass@4": 0.9997385302940074, + "pass@8": 0.9999862846484304, + "pass@16": 0.9999999645862426, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9994023075585583, + "pass@4": 0.9998913907449267, + "pass@8": 0.9999968657441765, + "pass@16": 0.9999999985924529, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972310126582279, + "pass@2": 0.9978734239509618, + "pass@4": 0.9985819919266421, + "pass@8": 0.9993828519658078, + "pass@16": 0.999893854092864, + "pass@32": 0.9999980312535122, + "pass@64": 0.999999999966339, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.987331081081081, + "pass@2": 0.9915873058097466, + "pass@4": 0.9947041267927096, + "pass@8": 0.997053024555423, + "pass@16": 0.9984133373783722, + "pass@32": 0.999305885850076, + "pass@64": 0.9999134567636024, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9977312359166519, + "pass@4": 0.9990373822846098, + "pass@8": 0.9997493167484548, + "pass@16": 0.9999841910297439, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9537202380952381, + "pass@2": 0.9642443132108488, + "pass@4": 0.972526728206593, + "pass@8": 0.9786626463839903, + "pass@16": 0.9826943761181406, + "pass@32": 0.9855854900825515, + "pass@64": 0.9870817812207828, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.9236190749235474, + "pass@2": 0.9405976095017944, + "pass@4": 0.9509045658283538, + "pass@8": 0.9587264846479097, + "pass@16": 0.9659104577653655, + "pass@32": 0.9739652412486856, + "pass@64": 0.9825721937453857, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.8131028163580247, + "pass@2": 0.8544848777583361, + "pass@4": 0.877677726337449, + "pass@8": 0.8922668036744839, + "pass@16": 0.9047770968835759, + "pass@32": 0.9171038736559595, + "pass@64": 0.9280859521752334, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8519566023738873, + "pass@2": 0.8872827052033923, + "pass@4": 0.9073185410429038, + "pass@8": 0.9216400776558572, + "pass@16": 0.9338163322916623, + "pass@32": 0.9445172996144986, + "pass@64": 0.9538269155955367, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.8259789737654321, + "pass@2": 0.8686012230242051, + "pass@4": 0.8924451495067742, + "pass@8": 0.9063586771341635, + "pass@16": 0.9160849619581726, + "pass@32": 0.9234598698400384, + "pass@64": 0.9293195664411008, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.7139795353982301, + "pass@2": 0.7931833292685756, + "pass@4": 0.8433599964827404, + "pass@8": 0.8761012011420178, + "pass@16": 0.8988421475917746, + "pass@32": 0.9149023008020526, + "pass@64": 0.9277295236545959, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.6482727786144579, + "pass@2": 0.7243408921117538, + "pass@4": 0.7755728016113949, + "pass@8": 0.8113793926723576, + "pass@16": 0.8388991160678905, + "pass@32": 0.8618016432288235, + "pass@64": 0.880137216239385, + "pass@128": 0.891566265060241 + }, + "18": { + "pass@1": 0.6368699596774193, + "pass@2": 0.7169791402082805, + "pass@4": 0.7700088709677413, + "pass@8": 0.8063951046233131, + "pass@16": 0.8307047111453599, + "pass@32": 0.848121899622024, + "pass@64": 0.8617897453414103, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.47365936555891236, + "pass@2": 0.5826961218688298, + "pass@4": 0.6706477077449906, + "pass@8": 0.7371792407322068, + "pass@16": 0.7865671010611726, + "pass@32": 0.8245143993387958, + "pass@64": 0.8530351330994856, + "pass@128": 0.8731117824773413 + }, + "20": { + "pass@1": 0.480762012012012, + "pass@2": 0.5811989400818144, + "pass@4": 0.659820350721925, + "pass@8": 0.7209768024610734, + "pass@16": 0.7675705711799523, + "pass@32": 0.8036268873233177, + "pass@64": 0.8330538100617901, + "pass@128": 0.8588588588588588 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..452f35d6e25c7f5dab6255306b435584b84ce6f6 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a3e387f661b4b89ccb33410855a0bcfba85bb7d83ed952b7da92fe1068b71d5 +size 1789314904 diff --git a/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json b/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6840f35ba5c3e88d13d1091ad7367a70a3012207 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op7-10", + "id": { + "answer_accuracy": 0.9950512152777777, + "avg_response_len": 200.65094791666667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9950512152777777, + "pass@2": 0.9968309820647415, + "pass@4": 0.9979443340311631, + "pass@8": 0.9986641931535145, + "pass@16": 0.99913249866909, + "pass@32": 0.9994619346745741, + "pass@64": 0.9996774367283783, + "pass@128": 0.9997777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9667265625, + "pass@2": 0.9772405265748035, + "pass@4": 0.9843605209036371, + "pass@8": 0.9892329969201861, + "pass@16": 0.9926182699254182, + "pass@32": 0.9952521952142179, + "pass@64": 0.9971000265839046, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999765625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99753125, + "pass@2": 0.9989874507874016, + "pass@4": 0.9995837750281215, + "pass@8": 0.9998582373960525, + "pass@16": 0.9999828674494142, + "pass@32": 0.9999998583460825, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996484375, + "pass@2": 0.999943282480315, + "pass@4": 0.9999983755155605, + "pass@8": 0.9999999988902694, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99934375, + "pass@2": 0.9998432578740157, + "pass@4": 0.9999854854705662, + "pass@8": 0.9999998491419404, + "pass@16": 0.9999999999930727, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958046875, + "pass@2": 0.9976202017716537, + "pass@4": 0.9989555868016498, + "pass@8": 0.9997269929148647, + "pass@16": 0.9999784812111003, + "pass@32": 0.9999999124700476, + "pass@64": 0.999999999999969, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9964296875, + "pass@2": 0.9978444881889764, + "pass@4": 0.9986152625609299, + "pass@8": 0.9991596631183184, + "pass@16": 0.9996128694428026, + "pass@32": 0.9999054460408194, + "pass@64": 0.9999969039717564, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9667265625, + "2": 1.0, + "3": 1.0, + "4": 0.9999765625, + "5": 0.99753125, + "6": 0.9996484375, + "7": 0.99934375, + "8": 0.9958046875, + "9": 0.9964296875 + }, + "per_op_avg_response_len": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377405, + "answer_accuracy": 0.9957705378250591, + "avg_response_len": 191.19166086203984, + "resp_tokens_sum": 72463169, + "pass_at_k": { + "pass@1": 0.9957705378250591, + "pass@2": 0.9972615208338319, + "pass@4": 0.9982072057038969, + "pass@8": 0.9988734692572885, + "pass@16": 0.9993439092985485, + "pass@32": 0.9996967349215862, + "pass@64": 0.9999100310573764, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9712389380530974, + "pass@2": 0.9810781914152326, + "pass@4": 0.9877872917987017, + "pass@8": 0.9923114067410685, + "pass@16": 0.9953009685125475, + "pass@32": 0.9976251357688136, + "pass@64": 0.9992232974200413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999267578125, + "pass@2": 0.9999988465797245, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967261904761905, + "pass@2": 0.9979080739907511, + "pass@4": 0.998781230917564, + "pass@8": 0.9995512519293239, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986362795857988, + "pass@2": 0.9996065176117038, + "pass@4": 0.9999584012797217, + "pass@8": 0.9999995539350579, + "pass@16": 0.9999999999795047, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9996494391025641, + "pass@2": 0.9999948736876642, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967649647887324, + "pass@2": 0.997464164910724, + "pass@4": 0.99782990673701, + "pass@8": 0.9983444563318933, + "pass@16": 0.9990631538364027, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388710, + "answer_accuracy": 0.9950186353211009, + "avg_response_len": 198.65210568889253, + "resp_tokens_sum": 77604637, + "pass_at_k": { + "pass@1": 0.9950186353211009, + "pass@2": 0.9969053063822872, + "pass@4": 0.9981152307018307, + "pass@8": 0.9988235206184004, + "pass@16": 0.999204425400664, + "pass@32": 0.999462311011784, + "pass@64": 0.9996276085051239, + "pass@128": 0.9996723460026212 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639810199386503, + "pass@2": 0.9763013411187863, + "pass@4": 0.9849948809121252, + "pass@8": 0.9900287848894771, + "pass@16": 0.9927408931048474, + "pass@32": 0.9949717791311338, + "pass@64": 0.9965136850356024, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995018115942029, + "pass@2": 0.9999390191715166, + "pass@4": 0.9999989468707715, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993955945558739, + "pass@2": 0.9998854291225775, + "pass@4": 0.9999959845635341, + "pass@8": 0.9999999968689766, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995484104046243, + "pass@2": 0.9999324393518729, + "pass@4": 0.9999986873931221, + "pass@8": 0.999999999745349, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9976322501940778, + "pass@4": 0.9990125788853857, + "pass@8": 0.9997359054080507, + "pass@16": 0.9999800133300545, + "pass@32": 0.9999999433844207, + "pass@64": 0.9999999999999964, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961171407185628, + "pass@2": 0.9976218822198123, + "pass@4": 0.9984793601772832, + "pass@8": 0.9992627421174618, + "pass@16": 0.999836737839751, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380184, + "answer_accuracy": 0.994371442919317, + "avg_response_len": 212.07023665048544, + "resp_tokens_sum": 81082086, + "pass_at_k": { + "pass@1": 0.994371442919317, + "pass@2": 0.9963282491847879, + "pass@4": 0.9975091349478536, + "pass@8": 0.9982939440855182, + "pass@16": 0.9988494363126815, + "pass@32": 0.9992287936927978, + "pass@64": 0.9994977795235602, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648320895522388, + "pass@2": 0.9742709924785523, + "pass@4": 0.9802755158403708, + "pass@8": 0.9853434213761023, + "pass@16": 0.9897842102909985, + "pass@32": 0.9931238035577333, + "pass@64": 0.9955219923489672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962775735294118, + "pass@2": 0.9990218996062991, + "pass@4": 0.9999060312313901, + "pass@8": 0.9999988032093555, + "pass@16": 0.9999999998445908, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.9999472167714053, + "pass@4": 0.9999992962236186, + "pass@8": 0.999999999946361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998763844936709, + "pass@2": 0.9999988319794678, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926097972972973, + "pass@2": 0.9953824395261401, + "pass@4": 0.9979162804536821, + "pass@8": 0.9994617011862063, + "pass@16": 0.9999566861229163, + "pass@32": 0.9999997975032378, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9963826366559485, + "pass@2": 0.9985176880522572, + "pass@4": 0.9996576828619894, + "pass@8": 0.9999795024220703, + "pass@16": 0.9999999337376294, + "pass@32": 0.9999999999998973, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5002296875, + "avg_response_len": 344.50245, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5002296875, + "pass@2": 0.5728906865157404, + "pass@4": 0.6352997881983414, + "pass@8": 0.6874998054154, + "pass@16": 0.7300696915573064, + "pass@32": 0.7647618102242172, + "pass@64": 0.7939591669390739, + "pass@128": 0.8201 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9412421875, + "pass@2": 0.953652312992126, + "pass@4": 0.961258159355081, + "pass@8": 0.9671273765338197, + "pass@16": 0.9721198100900965, + "pass@32": 0.976499939558484, + "pass@64": 0.9798367422797393, + "pass@128": 0.982 + }, + "12": { + "pass@1": 0.8730078125, + "pass@2": 0.9054156003937017, + "pass@4": 0.9239353474878146, + "pass@8": 0.936777551487186, + "pass@16": 0.9463443459241828, + "pass@32": 0.9542127970258184, + "pass@64": 0.9614905316338895, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.7339765625, + "pass@2": 0.7970696358267721, + "pass@4": 0.8382950806149229, + "pass@8": 0.8688697091343058, + "pass@16": 0.892709005541433, + "pass@32": 0.9105660402406084, + "pass@64": 0.923394088323559, + "pass@128": 0.935 + }, + "14": { + "pass@1": 0.677140625, + "pass@2": 0.7601444389763784, + "pass@4": 0.8148952628421446, + "pass@8": 0.8502713306521064, + "pass@16": 0.8758396736542128, + "pass@32": 0.8970577055422668, + "pass@64": 0.9145199374295602, + "pass@128": 0.928 + }, + "15": { + "pass@1": 0.5425546875, + "pass@2": 0.6452209645669293, + "pass@4": 0.7236853342707167, + "pass@8": 0.7831448421952832, + "pass@16": 0.8276656391456853, + "pass@32": 0.8600596388464278, + "pass@64": 0.8842948168837554, + "pass@128": 0.904 + }, + "16": { + "pass@1": 0.3742421875, + "pass@2": 0.482216658464567, + "pass@4": 0.5768006977877758, + "pass@8": 0.65205259160322, + "pass@16": 0.7090840741147838, + "pass@32": 0.754135531429707, + "pass@64": 0.7930790544821106, + "pass@128": 0.829 + }, + "17": { + "pass@1": 0.283015625, + "pass@2": 0.38146296751968556, + "pass@4": 0.4753989731908505, + "pass@8": 0.556040102428998, + "pass@16": 0.6219512049905577, + "pass@32": 0.6764085805789557, + "pass@64": 0.7233771012589468, + "pass@128": 0.766 + }, + "18": { + "pass@1": 0.2356953125, + "pass@2": 0.31902374507874, + "pass@4": 0.4022334073865767, + "pass@8": 0.4796290143755297, + "pass@16": 0.5462268840032527, + "pass@32": 0.600682493693144, + "pass@64": 0.6471233077378016, + "pass@128": 0.692 + }, + "19": { + "pass@1": 0.178984375, + "pass@2": 0.2540275590551181, + "pass@4": 0.33239348884514397, + "pass@8": 0.4048698789485252, + "pass@16": 0.46788108259685424, + "pass@32": 0.5236021150346005, + "pass@64": 0.5747571384612385, + "pass@128": 0.624 + }, + "20": { + "pass@1": 0.1624375, + "pass@2": 0.23067298228346464, + "pass@4": 0.30410213020247473, + "pass@8": 0.37621565679502567, + "pass@16": 0.4408751955118752, + "pass@32": 0.4943932602921467, + "pass@64": 0.537718950900034, + "pass@128": 0.573 + } + }, + "per_op_accuracy": { + "11": 0.9412421875, + "12": 0.8730078125, + "13": 0.7339765625, + "14": 0.677140625, + "15": 0.5425546875, + "16": 0.3742421875, + "17": 0.283015625, + "18": 0.2356953125, + "19": 0.178984375, + "20": 0.1624375 + }, + "per_op_avg_response_len": { + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 215464, + "answer_accuracy": 0.500241456166419, + "avg_response_len": 328.1673616270431, + "resp_tokens_sum": 141348246, + "pass_at_k": { + "pass@1": 0.500241456166419, + "pass@2": 0.5732799355921903, + "pass@4": 0.6357232889908101, + "pass@8": 0.6875347017891723, + "pass@16": 0.729570207801191, + "pass@32": 0.7639155179011134, + "pass@64": 0.7932490218591338, + "pass@128": 0.8199108469539376 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9256595254957507, + "pass@2": 0.9379046446655216, + "pass@4": 0.9464527073103111, + "pass@8": 0.9544125837859465, + "pass@16": 0.9620550938845545, + "pass@32": 0.9689466009864247, + "pass@64": 0.9733538467202176, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8546963276836158, + "pass@2": 0.8892605320521376, + "pass@4": 0.9096035241357541, + "pass@8": 0.9232561725273154, + "pass@16": 0.9335660780959657, + "pass@32": 0.9430045267513955, + "pass@64": 0.9524606602559231, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7480407523510971, + "pass@2": 0.8101477457853037, + "pass@4": 0.8495423074074989, + "pass@8": 0.8771021172653639, + "pass@16": 0.896003390136595, + "pass@32": 0.9090301477413725, + "pass@64": 0.9191436054892381, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682583722741433, + "pass@2": 0.7649933616650716, + "pass@4": 0.818672544436618, + "pass@8": 0.852279360455284, + "pass@16": 0.8766875266221614, + "pass@32": 0.8978875992415474, + "pass@64": 0.9145590851330114, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5563583815028902, + "pass@2": 0.6612466433025347, + "pass@4": 0.7391046363062996, + "pass@8": 0.7995924425216887, + "pass@16": 0.8472066555948774, + "pass@32": 0.8814056698555999, + "pass@64": 0.9055910973040048, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.35776209677419357, + "pass@2": 0.4704236252222501, + "pass@4": 0.5702631176143302, + "pass@8": 0.6486250520290218, + "pass@16": 0.7058972081786804, + "pass@32": 0.7489029331029314, + "pass@64": 0.7864609089054653, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.2822209176300578, + "pass@2": 0.38103423262482355, + "pass@4": 0.4747528679579794, + "pass@8": 0.5555152783820172, + "pass@16": 0.6224005820967119, + "pass@32": 0.6793292843272832, + "pass@64": 0.727092303936004, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.21580357142857143, + "pass@2": 0.29927622328458936, + "pass@4": 0.3840758931919225, + "pass@8": 0.46175944378352485, + "pass@16": 0.5272364483452201, + "pass@32": 0.5804867351029614, + "pass@64": 0.6281698996492903, + "pass@128": 0.68 + }, + "19": { + "pass@1": 0.18041478978978978, + "pass@2": 0.25600452815019736, + "pass@4": 0.3330379324769089, + "pass@8": 0.40319579360003893, + "pass@16": 0.46454362983819264, + "pass@32": 0.5196802274780482, + "pass@64": 0.5703849188444975, + "pass@128": 0.6186186186186187 + }, + "20": { + "pass@1": 0.17896021021021022, + "pass@2": 0.24850367099382842, + "pass@4": 0.32194917217555025, + "pass@8": 0.3922391954982608, + "pass@16": 0.45386218477737117, + "pass@32": 0.5048449014642736, + "pass@64": 0.55023639343001, + "pass@128": 0.5945945945945946 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 217083, + "answer_accuracy": 0.5043000111507583, + "avg_response_len": 340.7973303226286, + "resp_tokens_sum": 146700982, + "pass_at_k": { + "pass@1": 0.5043000111507583, + "pass@2": 0.5766524999356145, + "pass@4": 0.6387109358598265, + "pass@8": 0.6908086380944928, + "pass@16": 0.7329800101268554, + "pass@32": 0.7668444456595922, + "pass@64": 0.7957990950959042, + "pass@128": 0.8230746357418971 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9549839984939759, + "pass@2": 0.9656675422161087, + "pass@4": 0.9699165173065714, + "pass@8": 0.9719722005758548, + "pass@16": 0.9736488700375456, + "pass@32": 0.9754268332917806, + "pass@64": 0.9774205766085304, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8691222570532915, + "pass@2": 0.8991303717325303, + "pass@4": 0.9174640042055867, + "pass@8": 0.9318582991709181, + "pass@16": 0.9426732941451575, + "pass@32": 0.949884708573006, + "pass@64": 0.9548453788380237, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7502626050420168, + "pass@2": 0.8138757471492536, + "pass@4": 0.8540254450861707, + "pass@8": 0.883486764099256, + "pass@16": 0.9069851064183747, + "pass@32": 0.9246218794102262, + "pass@64": 0.9372843805980681, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6877741228070176, + "pass@2": 0.7709750713726572, + "pass@4": 0.8255886059746912, + "pass@8": 0.8616636529389289, + "pass@16": 0.8884648994522174, + "pass@32": 0.9102203876391841, + "pass@64": 0.9280497250391677, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.5294981060606061, + "pass@2": 0.6371901843235506, + "pass@4": 0.7218647379304856, + "pass@8": 0.784792093362903, + "pass@16": 0.8296342586994554, + "pass@32": 0.8615023805667635, + "pass@64": 0.8850014181350153, + "pass@128": 0.9030303030303031 + }, + "16": { + "pass@1": 0.38826566951566954, + "pass@2": 0.49237134620992856, + "pass@4": 0.5828217968480429, + "pass@8": 0.6537934355879218, + "pass@16": 0.7056221038183779, + "pass@32": 0.7460727057171335, + "pass@64": 0.784026806543782, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3012179736024845, + "pass@2": 0.40523403494400156, + "pass@4": 0.5060908428760066, + "pass@8": 0.5919590185783913, + "pass@16": 0.6587760745115138, + "pass@32": 0.7099266521318334, + "pass@64": 0.7537711568802992, + "pass@128": 0.7950310559006211 + }, + "18": { + "pass@1": 0.24391084558823528, + "pass@2": 0.32872459761463624, + "pass@4": 0.4134082233287015, + "pass@8": 0.4939817269103554, + "pass@16": 0.5646029865362203, + "pass@32": 0.6231760486863247, + "pass@64": 0.672253518753805, + "pass@128": 0.7176470588235294 + }, + "19": { + "pass@1": 0.17578125, + "pass@2": 0.24372283054461924, + "pass@4": 0.3149261088457693, + "pass@8": 0.382738789933009, + "pass@16": 0.44227697220974477, + "pass@32": 0.49650961347677647, + "pass@64": 0.5534303542394388, + "pass@128": 0.6190476190476191 + }, + "20": { + "pass@1": 0.14549026946107785, + "pass@2": 0.21075847812249524, + "pass@4": 0.28126393697419555, + "pass@8": 0.35182762438965187, + "pass@16": 0.41691692179414386, + "pass@32": 0.47061910952622776, + "pass@64": 0.511055493082766, + "pass@128": 0.5419161676646707 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 207747, + "answer_accuracy": 0.4960340579156479, + "avg_response_len": 365.10999579767724, + "resp_tokens_sum": 152913908, + "pass_at_k": { + "pass@1": 0.4960340579156479, + "pass@2": 0.5686239378381132, + "pass@4": 0.6313582326506261, + "pass@8": 0.684063060122764, + "pass@16": 0.7275921125505347, + "pass@32": 0.7634915995573603, + "pass@64": 0.7927983967075446, + "pass@128": 0.8172371638141809 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9442212301587302, + "pass@2": 0.9586360298712663, + "pass@4": 0.9687240315198693, + "pass@8": 0.9762697265593556, + "pass@16": 0.9817871018932806, + "pass@32": 0.9860954944679531, + "pass@64": 0.9896482633443495, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.896621750764526, + "pass@2": 0.9290360350598379, + "pass@4": 0.9457635553583317, + "pass@8": 0.956214278217107, + "pass@16": 0.9637589402015467, + "pass@32": 0.9705687233059196, + "pass@64": 0.977748568972364, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7021846064814815, + "pass@2": 0.7656755036696803, + "pass@4": 0.8098888909140981, + "pass@8": 0.8446585152568515, + "pass@16": 0.8737353120262339, + "pass@32": 0.8965907782767284, + "pass@64": 0.9122739330215466, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.6611646884272997, + "pass@2": 0.7445344079300921, + "pass@4": 0.8004453199841117, + "pass@8": 0.8367972897354505, + "pass@16": 0.86221953114493, + "pass@32": 0.882909239208098, + "pass@64": 0.9007521220726056, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.5411120756172839, + "pass@2": 0.6362866208564207, + "pass@4": 0.709073353678012, + "pass@8": 0.7639026736821625, + "pass@16": 0.80479268808345, + "pass@32": 0.8357947268190055, + "pass@64": 0.8608328062716483, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.37479258849557523, + "pass@2": 0.4824867024365314, + "pass@4": 0.5765447806302972, + "pass@8": 0.6533844530467929, + "pass@16": 0.7155828352187087, + "pass@32": 0.7672687388232577, + "pass@64": 0.8085037275060418, + "pass@128": 0.8407079646017699 + }, + "17": { + "pass@1": 0.26618975903614456, + "pass@2": 0.3588547101792997, + "pass@4": 0.4463049080461327, + "pass@8": 0.5217500365258368, + "pass@16": 0.5857671915433449, + "pass@32": 0.6408562235399593, + "pass@64": 0.6900266613904555, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.24914314516129032, + "pass@2": 0.33067968948437876, + "pass@4": 0.41047757689562997, + "pass@8": 0.4840626512960474, + "pass@16": 0.5475132634197127, + "pass@32": 0.5988138059798621, + "pass@64": 0.6409601499492135, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.18079682779456194, + "pass@2": 0.2624990335894569, + "pass@4": 0.34947638899744776, + "pass@8": 0.4290194751124504, + "pass@16": 0.49722958065940137, + "pass@32": 0.555049453644149, + "pass@64": 0.6008047174065532, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1629129129129129, + "pass@2": 0.2328166010498687, + "pass@4": 0.309161864620526, + "pass@8": 0.3846533878317151, + "pass@16": 0.451918426732032, + "pass@32": 0.5077871637320823, + "pass@64": 0.5519450366312206, + "pass@128": 0.5825825825825826 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7346188322368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7346188322368421, + "pass@2": 0.773704510723148, + "pass@4": 0.8070787835928236, + "pass@8": 0.8348934627650431, + "pass@16": 0.8575204949260501, + "pass@32": 0.8759355533849245, + "pass@64": 0.8914046631550582, + "pass@128": 0.9052105263157895 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9667265625, + "pass@2": 0.9772405265748035, + "pass@4": 0.9843605209036371, + "pass@8": 0.9892329969201861, + "pass@16": 0.9926182699254182, + "pass@32": 0.9952521952142179, + "pass@64": 0.9971000265839046, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999765625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99753125, + "pass@2": 0.9989874507874016, + "pass@4": 0.9995837750281215, + "pass@8": 0.9998582373960525, + "pass@16": 0.9999828674494142, + "pass@32": 0.9999998583460825, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996484375, + "pass@2": 0.999943282480315, + "pass@4": 0.9999983755155605, + "pass@8": 0.9999999988902694, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99934375, + "pass@2": 0.9998432578740157, + "pass@4": 0.9999854854705662, + "pass@8": 0.9999998491419404, + "pass@16": 0.9999999999930727, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958046875, + "pass@2": 0.9976202017716537, + "pass@4": 0.9989555868016498, + "pass@8": 0.9997269929148647, + "pass@16": 0.9999784812111003, + "pass@32": 0.9999999124700476, + "pass@64": 0.999999999999969, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9964296875, + "pass@2": 0.9978444881889764, + "pass@4": 0.9986152625609299, + "pass@8": 0.9991596631183184, + "pass@16": 0.9996128694428026, + "pass@32": 0.9999054460408194, + "pass@64": 0.9999969039717564, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9412421875, + "pass@2": 0.953652312992126, + "pass@4": 0.961258159355081, + "pass@8": 0.9671273765338197, + "pass@16": 0.9721198100900965, + "pass@32": 0.976499939558484, + "pass@64": 0.9798367422797393, + "pass@128": 0.982 + }, + "12": { + "pass@1": 0.8730078125, + "pass@2": 0.9054156003937017, + "pass@4": 0.9239353474878146, + "pass@8": 0.936777551487186, + "pass@16": 0.9463443459241828, + "pass@32": 0.9542127970258184, + "pass@64": 0.9614905316338895, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.7339765625, + "pass@2": 0.7970696358267721, + "pass@4": 0.8382950806149229, + "pass@8": 0.8688697091343058, + "pass@16": 0.892709005541433, + "pass@32": 0.9105660402406084, + "pass@64": 0.923394088323559, + "pass@128": 0.935 + }, + "14": { + "pass@1": 0.677140625, + "pass@2": 0.7601444389763784, + "pass@4": 0.8148952628421446, + "pass@8": 0.8502713306521064, + "pass@16": 0.8758396736542128, + "pass@32": 0.8970577055422668, + "pass@64": 0.9145199374295602, + "pass@128": 0.928 + }, + "15": { + "pass@1": 0.5425546875, + "pass@2": 0.6452209645669293, + "pass@4": 0.7236853342707167, + "pass@8": 0.7831448421952832, + "pass@16": 0.8276656391456853, + "pass@32": 0.8600596388464278, + "pass@64": 0.8842948168837554, + "pass@128": 0.904 + }, + "16": { + "pass@1": 0.3742421875, + "pass@2": 0.482216658464567, + "pass@4": 0.5768006977877758, + "pass@8": 0.65205259160322, + "pass@16": 0.7090840741147838, + "pass@32": 0.754135531429707, + "pass@64": 0.7930790544821106, + "pass@128": 0.829 + }, + "17": { + "pass@1": 0.283015625, + "pass@2": 0.38146296751968556, + "pass@4": 0.4753989731908505, + "pass@8": 0.556040102428998, + "pass@16": 0.6219512049905577, + "pass@32": 0.6764085805789557, + "pass@64": 0.7233771012589468, + "pass@128": 0.766 + }, + "18": { + "pass@1": 0.2356953125, + "pass@2": 0.31902374507874, + "pass@4": 0.4022334073865767, + "pass@8": 0.4796290143755297, + "pass@16": 0.5462268840032527, + "pass@32": 0.600682493693144, + "pass@64": 0.6471233077378016, + "pass@128": 0.692 + }, + "19": { + "pass@1": 0.178984375, + "pass@2": 0.2540275590551181, + "pass@4": 0.33239348884514397, + "pass@8": 0.4048698789485252, + "pass@16": 0.46788108259685424, + "pass@32": 0.5236021150346005, + "pass@64": 0.5747571384612385, + "pass@128": 0.624 + }, + "20": { + "pass@1": 0.1624375, + "pass@2": 0.23067298228346464, + "pass@4": 0.30410213020247473, + "pass@8": 0.37621565679502567, + "pass@16": 0.4408751955118752, + "pass@32": 0.4943932602921467, + "pass@64": 0.537718950900034, + "pass@128": 0.573 + } + }, + "per_op_accuracy": { + "10": 0.9667265625, + "2": 1.0, + "3": 1.0, + "4": 0.9999765625, + "5": 0.99753125, + "6": 0.9996484375, + "7": 0.99934375, + "8": 0.9958046875, + "9": 0.9964296875, + "11": 0.9412421875, + "12": 0.8730078125, + "13": 0.7339765625, + "14": 0.677140625, + "15": 0.5425546875, + "16": 0.3742421875, + "17": 0.283015625, + "18": 0.2356953125, + "19": 0.178984375, + "20": 0.1624375 + }, + "per_op_avg_response_len": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125, + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125, + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 592869, + "answer_accuracy": 0.7321829058646854, + "avg_response_len": 264.0533796534145, + "resp_tokens_sum": 213811415, + "pass_at_k": { + "pass@1": 0.7321829058646854, + "pass@2": 0.771732271017493, + "pass@4": 0.805390515893657, + "pass@8": 0.8332625061636697, + "pass@16": 0.8558427228397145, + "pass@32": 0.8742772288713363, + "pass@64": 0.8899804869612601, + "pass@128": 0.9042048687954474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9712389380530974, + "pass@2": 0.9810781914152326, + "pass@4": 0.9877872917987017, + "pass@8": 0.9923114067410685, + "pass@16": 0.9953009685125475, + "pass@32": 0.9976251357688136, + "pass@64": 0.9992232974200413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999267578125, + "pass@2": 0.9999988465797245, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967261904761905, + "pass@2": 0.9979080739907511, + "pass@4": 0.998781230917564, + "pass@8": 0.9995512519293239, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986362795857988, + "pass@2": 0.9996065176117038, + "pass@4": 0.9999584012797217, + "pass@8": 0.9999995539350579, + "pass@16": 0.9999999999795047, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9996494391025641, + "pass@2": 0.9999948736876642, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967649647887324, + "pass@2": 0.997464164910724, + "pass@4": 0.99782990673701, + "pass@8": 0.9983444563318933, + "pass@16": 0.9990631538364027, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9256595254957507, + "pass@2": 0.9379046446655216, + "pass@4": 0.9464527073103111, + "pass@8": 0.9544125837859465, + "pass@16": 0.9620550938845545, + "pass@32": 0.9689466009864247, + "pass@64": 0.9733538467202176, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8546963276836158, + "pass@2": 0.8892605320521376, + "pass@4": 0.9096035241357541, + "pass@8": 0.9232561725273154, + "pass@16": 0.9335660780959657, + "pass@32": 0.9430045267513955, + "pass@64": 0.9524606602559231, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7480407523510971, + "pass@2": 0.8101477457853037, + "pass@4": 0.8495423074074989, + "pass@8": 0.8771021172653639, + "pass@16": 0.896003390136595, + "pass@32": 0.9090301477413725, + "pass@64": 0.9191436054892381, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682583722741433, + "pass@2": 0.7649933616650716, + "pass@4": 0.818672544436618, + "pass@8": 0.852279360455284, + "pass@16": 0.8766875266221614, + "pass@32": 0.8978875992415474, + "pass@64": 0.9145590851330114, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5563583815028902, + "pass@2": 0.6612466433025347, + "pass@4": 0.7391046363062996, + "pass@8": 0.7995924425216887, + "pass@16": 0.8472066555948774, + "pass@32": 0.8814056698555999, + "pass@64": 0.9055910973040048, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.35776209677419357, + "pass@2": 0.4704236252222501, + "pass@4": 0.5702631176143302, + "pass@8": 0.6486250520290218, + "pass@16": 0.7058972081786804, + "pass@32": 0.7489029331029314, + "pass@64": 0.7864609089054653, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.2822209176300578, + "pass@2": 0.38103423262482355, + "pass@4": 0.4747528679579794, + "pass@8": 0.5555152783820172, + "pass@16": 0.6224005820967119, + "pass@32": 0.6793292843272832, + "pass@64": 0.727092303936004, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.21580357142857143, + "pass@2": 0.29927622328458936, + "pass@4": 0.3840758931919225, + "pass@8": 0.46175944378352485, + "pass@16": 0.5272364483452201, + "pass@32": 0.5804867351029614, + "pass@64": 0.6281698996492903, + "pass@128": 0.68 + }, + "19": { + "pass@1": 0.18041478978978978, + "pass@2": 0.25600452815019736, + "pass@4": 0.3330379324769089, + "pass@8": 0.40319579360003893, + "pass@16": 0.46454362983819264, + "pass@32": 0.5196802274780482, + "pass@64": 0.5703849188444975, + "pass@128": 0.6186186186186187 + }, + "20": { + "pass@1": 0.17896021021021022, + "pass@2": 0.24850367099382842, + "pass@4": 0.32194917217555025, + "pass@8": 0.3922391954982608, + "pass@16": 0.45386218477737117, + "pass@32": 0.5048449014642736, + "pass@64": 0.55023639343001, + "pass@128": 0.5945945945945946 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605793, + "answer_accuracy": 0.737764273187841, + "avg_response_len": 273.1703271141855, + "resp_tokens_sum": 224305619, + "pass_at_k": { + "pass@1": 0.737764273187841, + "pass@2": 0.7765919489262912, + "pass@4": 0.8097011007636077, + "pass@8": 0.8373497793981471, + "pass@16": 0.8596389213374069, + "pass@32": 0.8775147067749307, + "pass@64": 0.8927725359259874, + "pass@128": 0.9070927513639907 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639810199386503, + "pass@2": 0.9763013411187863, + "pass@4": 0.9849948809121252, + "pass@8": 0.9900287848894771, + "pass@16": 0.9927408931048474, + "pass@32": 0.9949717791311338, + "pass@64": 0.9965136850356024, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995018115942029, + "pass@2": 0.9999390191715166, + "pass@4": 0.9999989468707715, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993955945558739, + "pass@2": 0.9998854291225775, + "pass@4": 0.9999959845635341, + "pass@8": 0.9999999968689766, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995484104046243, + "pass@2": 0.9999324393518729, + "pass@4": 0.9999986873931221, + "pass@8": 0.999999999745349, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9976322501940778, + "pass@4": 0.9990125788853857, + "pass@8": 0.9997359054080507, + "pass@16": 0.9999800133300545, + "pass@32": 0.9999999433844207, + "pass@64": 0.9999999999999964, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961171407185628, + "pass@2": 0.9976218822198123, + "pass@4": 0.9984793601772832, + "pass@8": 0.9992627421174618, + "pass@16": 0.999836737839751, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9549839984939759, + "pass@2": 0.9656675422161087, + "pass@4": 0.9699165173065714, + "pass@8": 0.9719722005758548, + "pass@16": 0.9736488700375456, + "pass@32": 0.9754268332917806, + "pass@64": 0.9774205766085304, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8691222570532915, + "pass@2": 0.8991303717325303, + "pass@4": 0.9174640042055867, + "pass@8": 0.9318582991709181, + "pass@16": 0.9426732941451575, + "pass@32": 0.949884708573006, + "pass@64": 0.9548453788380237, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7502626050420168, + "pass@2": 0.8138757471492536, + "pass@4": 0.8540254450861707, + "pass@8": 0.883486764099256, + "pass@16": 0.9069851064183747, + "pass@32": 0.9246218794102262, + "pass@64": 0.9372843805980681, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6877741228070176, + "pass@2": 0.7709750713726572, + "pass@4": 0.8255886059746912, + "pass@8": 0.8616636529389289, + "pass@16": 0.8884648994522174, + "pass@32": 0.9102203876391841, + "pass@64": 0.9280497250391677, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.5294981060606061, + "pass@2": 0.6371901843235506, + "pass@4": 0.7218647379304856, + "pass@8": 0.784792093362903, + "pass@16": 0.8296342586994554, + "pass@32": 0.8615023805667635, + "pass@64": 0.8850014181350153, + "pass@128": 0.9030303030303031 + }, + "16": { + "pass@1": 0.38826566951566954, + "pass@2": 0.49237134620992856, + "pass@4": 0.5828217968480429, + "pass@8": 0.6537934355879218, + "pass@16": 0.7056221038183779, + "pass@32": 0.7460727057171335, + "pass@64": 0.784026806543782, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3012179736024845, + "pass@2": 0.40523403494400156, + "pass@4": 0.5060908428760066, + "pass@8": 0.5919590185783913, + "pass@16": 0.6587760745115138, + "pass@32": 0.7099266521318334, + "pass@64": 0.7537711568802992, + "pass@128": 0.7950310559006211 + }, + "18": { + "pass@1": 0.24391084558823528, + "pass@2": 0.32872459761463624, + "pass@4": 0.4134082233287015, + "pass@8": 0.4939817269103554, + "pass@16": 0.5646029865362203, + "pass@32": 0.6231760486863247, + "pass@64": 0.672253518753805, + "pass@128": 0.7176470588235294 + }, + "19": { + "pass@1": 0.17578125, + "pass@2": 0.24372283054461924, + "pass@4": 0.3149261088457693, + "pass@8": 0.382738789933009, + "pass@16": 0.44227697220974477, + "pass@32": 0.49650961347677647, + "pass@64": 0.5534303542394388, + "pass@128": 0.6190476190476191 + }, + "20": { + "pass@1": 0.14549026946107785, + "pass@2": 0.21075847812249524, + "pass@4": 0.28126393697419555, + "pass@8": 0.35182762438965187, + "pass@16": 0.41691692179414386, + "pass@32": 0.47061910952622776, + "pass@64": 0.511055493082766, + "pass@128": 0.5419161676646707 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587931, + "answer_accuracy": 0.7338569959258667, + "avg_response_len": 292.07440535628695, + "resp_tokens_sum": 233995994, + "pass_at_k": { + "pass@1": 0.7338569959258667, + "pass@2": 0.7727384574087262, + "pass@4": 0.80609744740726, + "pass@8": 0.8340243399432995, + "pass@16": 0.8570450005642073, + "pass@32": 0.8759931171931754, + "pass@64": 0.8914421187831943, + "pass@128": 0.9042978111519412 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648320895522388, + "pass@2": 0.9742709924785523, + "pass@4": 0.9802755158403708, + "pass@8": 0.9853434213761023, + "pass@16": 0.9897842102909985, + "pass@32": 0.9931238035577333, + "pass@64": 0.9955219923489672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962775735294118, + "pass@2": 0.9990218996062991, + "pass@4": 0.9999060312313901, + "pass@8": 0.9999988032093555, + "pass@16": 0.9999999998445908, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.9999472167714053, + "pass@4": 0.9999992962236186, + "pass@8": 0.999999999946361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998763844936709, + "pass@2": 0.9999988319794678, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926097972972973, + "pass@2": 0.9953824395261401, + "pass@4": 0.9979162804536821, + "pass@8": 0.9994617011862063, + "pass@16": 0.9999566861229163, + "pass@32": 0.9999997975032378, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9963826366559485, + "pass@2": 0.9985176880522572, + "pass@4": 0.9996576828619894, + "pass@8": 0.9999795024220703, + "pass@16": 0.9999999337376294, + "pass@32": 0.9999999999998973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9442212301587302, + "pass@2": 0.9586360298712663, + "pass@4": 0.9687240315198693, + "pass@8": 0.9762697265593556, + "pass@16": 0.9817871018932806, + "pass@32": 0.9860954944679531, + "pass@64": 0.9896482633443495, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.896621750764526, + "pass@2": 0.9290360350598379, + "pass@4": 0.9457635553583317, + "pass@8": 0.956214278217107, + "pass@16": 0.9637589402015467, + "pass@32": 0.9705687233059196, + "pass@64": 0.977748568972364, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7021846064814815, + "pass@2": 0.7656755036696803, + "pass@4": 0.8098888909140981, + "pass@8": 0.8446585152568515, + "pass@16": 0.8737353120262339, + "pass@32": 0.8965907782767284, + "pass@64": 0.9122739330215466, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.6611646884272997, + "pass@2": 0.7445344079300921, + "pass@4": 0.8004453199841117, + "pass@8": 0.8367972897354505, + "pass@16": 0.86221953114493, + "pass@32": 0.882909239208098, + "pass@64": 0.9007521220726056, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.5411120756172839, + "pass@2": 0.6362866208564207, + "pass@4": 0.709073353678012, + "pass@8": 0.7639026736821625, + "pass@16": 0.80479268808345, + "pass@32": 0.8357947268190055, + "pass@64": 0.8608328062716483, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.37479258849557523, + "pass@2": 0.4824867024365314, + "pass@4": 0.5765447806302972, + "pass@8": 0.6533844530467929, + "pass@16": 0.7155828352187087, + "pass@32": 0.7672687388232577, + "pass@64": 0.8085037275060418, + "pass@128": 0.8407079646017699 + }, + "17": { + "pass@1": 0.26618975903614456, + "pass@2": 0.3588547101792997, + "pass@4": 0.4463049080461327, + "pass@8": 0.5217500365258368, + "pass@16": 0.5857671915433449, + "pass@32": 0.6408562235399593, + "pass@64": 0.6900266613904555, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.24914314516129032, + "pass@2": 0.33067968948437876, + "pass@4": 0.41047757689562997, + "pass@8": 0.4840626512960474, + "pass@16": 0.5475132634197127, + "pass@32": 0.5988138059798621, + "pass@64": 0.6409601499492135, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.18079682779456194, + "pass@2": 0.2624990335894569, + "pass@4": 0.34947638899744776, + "pass@8": 0.4290194751124504, + "pass@16": 0.49722958065940137, + "pass@32": 0.555049453644149, + "pass@64": 0.6008047174065532, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1629129129129129, + "pass@2": 0.2328166010498687, + "pass@4": 0.309161864620526, + "pass@8": 0.3846533878317151, + "pass@16": 0.451918426732032, + "pass@32": 0.5077871637320823, + "pass@64": 0.5519450366312206, + "pass@128": 0.5825825825825826 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0eba835c3b45144f8af35ef5db2b26105624e5a9 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2804a617696ad2600619a0d6c58bb31804621104b524e41092418347b45c079a +size 2613776781 diff --git a/results/composition/SDAR-100M-block4-epoch4-op7-10/summary.csv b/results/composition/SDAR-100M-block4-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9824b208d15cb40f8cfdfb72a7ae37cbee158217 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op7-10,0.9950512152777777,0.5002296875,0.7346188322368421,0.0,0.0,0.0,200.65094791666667,344.50245,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch4-op7-10/summary.json b/results/composition/SDAR-100M-block4-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..abfc6c905c29ae01839c24ab7b70b51a1833e3c7 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op7-10", + "id_acc": 0.9950512152777777, + "ood_acc": 0.5002296875, + "total_acc": 0.7346188322368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.65094791666667, + "ood_avg_resp_len": 344.50245, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 592869, + "answer_accuracy": 0.7321829058646854, + "avg_response_len": 264.0533796534145, + "resp_tokens_sum": 213811415, + "pass_at_k": { + "pass@1": 0.7321829058646854, + "pass@2": 0.771732271017493, + "pass@4": 0.805390515893657, + "pass@8": 0.8332625061636697, + "pass@16": 0.8558427228397145, + "pass@32": 0.8742772288713363, + "pass@64": 0.8899804869612601, + "pass@128": 0.9042048687954474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9712389380530974, + "pass@2": 0.9810781914152326, + "pass@4": 0.9877872917987017, + "pass@8": 0.9923114067410685, + "pass@16": 0.9953009685125475, + "pass@32": 0.9976251357688136, + "pass@64": 0.9992232974200413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999267578125, + "pass@2": 0.9999988465797245, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967261904761905, + "pass@2": 0.9979080739907511, + "pass@4": 0.998781230917564, + "pass@8": 0.9995512519293239, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986362795857988, + "pass@2": 0.9996065176117038, + "pass@4": 0.9999584012797217, + "pass@8": 0.9999995539350579, + "pass@16": 0.9999999999795047, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9996494391025641, + "pass@2": 0.9999948736876642, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967649647887324, + "pass@2": 0.997464164910724, + "pass@4": 0.99782990673701, + "pass@8": 0.9983444563318933, + "pass@16": 0.9990631538364027, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9256595254957507, + "pass@2": 0.9379046446655216, + "pass@4": 0.9464527073103111, + "pass@8": 0.9544125837859465, + "pass@16": 0.9620550938845545, + "pass@32": 0.9689466009864247, + "pass@64": 0.9733538467202176, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8546963276836158, + "pass@2": 0.8892605320521376, + "pass@4": 0.9096035241357541, + "pass@8": 0.9232561725273154, + "pass@16": 0.9335660780959657, + "pass@32": 0.9430045267513955, + "pass@64": 0.9524606602559231, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7480407523510971, + "pass@2": 0.8101477457853037, + "pass@4": 0.8495423074074989, + "pass@8": 0.8771021172653639, + "pass@16": 0.896003390136595, + "pass@32": 0.9090301477413725, + "pass@64": 0.9191436054892381, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682583722741433, + "pass@2": 0.7649933616650716, + "pass@4": 0.818672544436618, + "pass@8": 0.852279360455284, + "pass@16": 0.8766875266221614, + "pass@32": 0.8978875992415474, + "pass@64": 0.9145590851330114, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5563583815028902, + "pass@2": 0.6612466433025347, + "pass@4": 0.7391046363062996, + "pass@8": 0.7995924425216887, + "pass@16": 0.8472066555948774, + "pass@32": 0.8814056698555999, + "pass@64": 0.9055910973040048, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.35776209677419357, + "pass@2": 0.4704236252222501, + "pass@4": 0.5702631176143302, + "pass@8": 0.6486250520290218, + "pass@16": 0.7058972081786804, + "pass@32": 0.7489029331029314, + "pass@64": 0.7864609089054653, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.2822209176300578, + "pass@2": 0.38103423262482355, + "pass@4": 0.4747528679579794, + "pass@8": 0.5555152783820172, + "pass@16": 0.6224005820967119, + "pass@32": 0.6793292843272832, + "pass@64": 0.727092303936004, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.21580357142857143, + "pass@2": 0.29927622328458936, + "pass@4": 0.3840758931919225, + "pass@8": 0.46175944378352485, + "pass@16": 0.5272364483452201, + "pass@32": 0.5804867351029614, + "pass@64": 0.6281698996492903, + "pass@128": 0.68 + }, + "19": { + "pass@1": 0.18041478978978978, + "pass@2": 0.25600452815019736, + "pass@4": 0.3330379324769089, + "pass@8": 0.40319579360003893, + "pass@16": 0.46454362983819264, + "pass@32": 0.5196802274780482, + "pass@64": 0.5703849188444975, + "pass@128": 0.6186186186186187 + }, + "20": { + "pass@1": 0.17896021021021022, + "pass@2": 0.24850367099382842, + "pass@4": 0.32194917217555025, + "pass@8": 0.3922391954982608, + "pass@16": 0.45386218477737117, + "pass@32": 0.5048449014642736, + "pass@64": 0.55023639343001, + "pass@128": 0.5945945945945946 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605793, + "answer_accuracy": 0.737764273187841, + "avg_response_len": 273.1703271141855, + "resp_tokens_sum": 224305619, + "pass_at_k": { + "pass@1": 0.737764273187841, + "pass@2": 0.7765919489262912, + "pass@4": 0.8097011007636077, + "pass@8": 0.8373497793981471, + "pass@16": 0.8596389213374069, + "pass@32": 0.8775147067749307, + "pass@64": 0.8927725359259874, + "pass@128": 0.9070927513639907 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639810199386503, + "pass@2": 0.9763013411187863, + "pass@4": 0.9849948809121252, + "pass@8": 0.9900287848894771, + "pass@16": 0.9927408931048474, + "pass@32": 0.9949717791311338, + "pass@64": 0.9965136850356024, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995018115942029, + "pass@2": 0.9999390191715166, + "pass@4": 0.9999989468707715, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993955945558739, + "pass@2": 0.9998854291225775, + "pass@4": 0.9999959845635341, + "pass@8": 0.9999999968689766, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995484104046243, + "pass@2": 0.9999324393518729, + "pass@4": 0.9999986873931221, + "pass@8": 0.999999999745349, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9976322501940778, + "pass@4": 0.9990125788853857, + "pass@8": 0.9997359054080507, + "pass@16": 0.9999800133300545, + "pass@32": 0.9999999433844207, + "pass@64": 0.9999999999999964, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961171407185628, + "pass@2": 0.9976218822198123, + "pass@4": 0.9984793601772832, + "pass@8": 0.9992627421174618, + "pass@16": 0.999836737839751, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9549839984939759, + "pass@2": 0.9656675422161087, + "pass@4": 0.9699165173065714, + "pass@8": 0.9719722005758548, + "pass@16": 0.9736488700375456, + "pass@32": 0.9754268332917806, + "pass@64": 0.9774205766085304, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8691222570532915, + "pass@2": 0.8991303717325303, + "pass@4": 0.9174640042055867, + "pass@8": 0.9318582991709181, + "pass@16": 0.9426732941451575, + "pass@32": 0.949884708573006, + "pass@64": 0.9548453788380237, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7502626050420168, + "pass@2": 0.8138757471492536, + "pass@4": 0.8540254450861707, + "pass@8": 0.883486764099256, + "pass@16": 0.9069851064183747, + "pass@32": 0.9246218794102262, + "pass@64": 0.9372843805980681, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6877741228070176, + "pass@2": 0.7709750713726572, + "pass@4": 0.8255886059746912, + "pass@8": 0.8616636529389289, + "pass@16": 0.8884648994522174, + "pass@32": 0.9102203876391841, + "pass@64": 0.9280497250391677, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.5294981060606061, + "pass@2": 0.6371901843235506, + "pass@4": 0.7218647379304856, + "pass@8": 0.784792093362903, + "pass@16": 0.8296342586994554, + "pass@32": 0.8615023805667635, + "pass@64": 0.8850014181350153, + "pass@128": 0.9030303030303031 + }, + "16": { + "pass@1": 0.38826566951566954, + "pass@2": 0.49237134620992856, + "pass@4": 0.5828217968480429, + "pass@8": 0.6537934355879218, + "pass@16": 0.7056221038183779, + "pass@32": 0.7460727057171335, + "pass@64": 0.784026806543782, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3012179736024845, + "pass@2": 0.40523403494400156, + "pass@4": 0.5060908428760066, + "pass@8": 0.5919590185783913, + "pass@16": 0.6587760745115138, + "pass@32": 0.7099266521318334, + "pass@64": 0.7537711568802992, + "pass@128": 0.7950310559006211 + }, + "18": { + "pass@1": 0.24391084558823528, + "pass@2": 0.32872459761463624, + "pass@4": 0.4134082233287015, + "pass@8": 0.4939817269103554, + "pass@16": 0.5646029865362203, + "pass@32": 0.6231760486863247, + "pass@64": 0.672253518753805, + "pass@128": 0.7176470588235294 + }, + "19": { + "pass@1": 0.17578125, + "pass@2": 0.24372283054461924, + "pass@4": 0.3149261088457693, + "pass@8": 0.382738789933009, + "pass@16": 0.44227697220974477, + "pass@32": 0.49650961347677647, + "pass@64": 0.5534303542394388, + "pass@128": 0.6190476190476191 + }, + "20": { + "pass@1": 0.14549026946107785, + "pass@2": 0.21075847812249524, + "pass@4": 0.28126393697419555, + "pass@8": 0.35182762438965187, + "pass@16": 0.41691692179414386, + "pass@32": 0.47061910952622776, + "pass@64": 0.511055493082766, + "pass@128": 0.5419161676646707 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587931, + "answer_accuracy": 0.7338569959258667, + "avg_response_len": 292.07440535628695, + "resp_tokens_sum": 233995994, + "pass_at_k": { + "pass@1": 0.7338569959258667, + "pass@2": 0.7727384574087262, + "pass@4": 0.80609744740726, + "pass@8": 0.8340243399432995, + "pass@16": 0.8570450005642073, + "pass@32": 0.8759931171931754, + "pass@64": 0.8914421187831943, + "pass@128": 0.9042978111519412 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648320895522388, + "pass@2": 0.9742709924785523, + "pass@4": 0.9802755158403708, + "pass@8": 0.9853434213761023, + "pass@16": 0.9897842102909985, + "pass@32": 0.9931238035577333, + "pass@64": 0.9955219923489672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962775735294118, + "pass@2": 0.9990218996062991, + "pass@4": 0.9999060312313901, + "pass@8": 0.9999988032093555, + "pass@16": 0.9999999998445908, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.9999472167714053, + "pass@4": 0.9999992962236186, + "pass@8": 0.999999999946361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998763844936709, + "pass@2": 0.9999988319794678, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926097972972973, + "pass@2": 0.9953824395261401, + "pass@4": 0.9979162804536821, + "pass@8": 0.9994617011862063, + "pass@16": 0.9999566861229163, + "pass@32": 0.9999997975032378, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9963826366559485, + "pass@2": 0.9985176880522572, + "pass@4": 0.9996576828619894, + "pass@8": 0.9999795024220703, + "pass@16": 0.9999999337376294, + "pass@32": 0.9999999999998973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9442212301587302, + "pass@2": 0.9586360298712663, + "pass@4": 0.9687240315198693, + "pass@8": 0.9762697265593556, + "pass@16": 0.9817871018932806, + "pass@32": 0.9860954944679531, + "pass@64": 0.9896482633443495, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.896621750764526, + "pass@2": 0.9290360350598379, + "pass@4": 0.9457635553583317, + "pass@8": 0.956214278217107, + "pass@16": 0.9637589402015467, + "pass@32": 0.9705687233059196, + "pass@64": 0.977748568972364, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7021846064814815, + "pass@2": 0.7656755036696803, + "pass@4": 0.8098888909140981, + "pass@8": 0.8446585152568515, + "pass@16": 0.8737353120262339, + "pass@32": 0.8965907782767284, + "pass@64": 0.9122739330215466, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.6611646884272997, + "pass@2": 0.7445344079300921, + "pass@4": 0.8004453199841117, + "pass@8": 0.8367972897354505, + "pass@16": 0.86221953114493, + "pass@32": 0.882909239208098, + "pass@64": 0.9007521220726056, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.5411120756172839, + "pass@2": 0.6362866208564207, + "pass@4": 0.709073353678012, + "pass@8": 0.7639026736821625, + "pass@16": 0.80479268808345, + "pass@32": 0.8357947268190055, + "pass@64": 0.8608328062716483, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.37479258849557523, + "pass@2": 0.4824867024365314, + "pass@4": 0.5765447806302972, + "pass@8": 0.6533844530467929, + "pass@16": 0.7155828352187087, + "pass@32": 0.7672687388232577, + "pass@64": 0.8085037275060418, + "pass@128": 0.8407079646017699 + }, + "17": { + "pass@1": 0.26618975903614456, + "pass@2": 0.3588547101792997, + "pass@4": 0.4463049080461327, + "pass@8": 0.5217500365258368, + "pass@16": 0.5857671915433449, + "pass@32": 0.6408562235399593, + "pass@64": 0.6900266613904555, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.24914314516129032, + "pass@2": 0.33067968948437876, + "pass@4": 0.41047757689562997, + "pass@8": 0.4840626512960474, + "pass@16": 0.5475132634197127, + "pass@32": 0.5988138059798621, + "pass@64": 0.6409601499492135, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.18079682779456194, + "pass@2": 0.2624990335894569, + "pass@4": 0.34947638899744776, + "pass@8": 0.4290194751124504, + "pass@16": 0.49722958065940137, + "pass@32": 0.555049453644149, + "pass@64": 0.6008047174065532, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1629129129129129, + "pass@2": 0.2328166010498687, + "pass@4": 0.309161864620526, + "pass@8": 0.3846533878317151, + "pass@16": 0.451918426732032, + "pass@32": 0.5077871637320823, + "pass@64": 0.5519450366312206, + "pass@128": 0.5825825825825826 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4e8d74e34e2c37dfbef1852b0b0521adba682053 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f53cb79b86d6b802fd62e944e00aa33c1a44867910f8e9272985aa65d62bf5c +size 1789436772 diff --git a/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json b/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..60f607596603c58159fa8d65c9e7ac501e9a415c --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9942543402777778, + "avg_response_len": 200.17993142361112, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9942543402777778, + "pass@2": 0.9962128034776896, + "pass@4": 0.9975820776257135, + "pass@8": 0.9984810237584633, + "pass@16": 0.9990155200886814, + "pass@32": 0.9993061579381566, + "pass@64": 0.9994829050729563, + "pass@128": 0.9996666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.964, + "pass@2": 0.9748387057086616, + "pass@4": 0.9833349187289088, + "pass@8": 0.9891501055647638, + "pass@16": 0.9926545771584487, + "pass@32": 0.99481551787825, + "pass@64": 0.9963468166102103, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996796875, + "pass@2": 0.9999547244094489, + "pass@4": 0.9999985867079115, + "pass@8": 0.9999999989072665, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966875, + "pass@2": 0.9983604822834645, + "pass@4": 0.999337447225347, + "pass@8": 0.9997908682563258, + "pass@16": 0.999966397971859, + "pass@32": 0.9999993778758997, + "pass@64": 0.9999999999893631, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992578125, + "pass@2": 0.9997546751968504, + "pass@4": 0.9999684317585302, + "pass@8": 0.999999446635866, + "pass@16": 0.9999999998888393, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984375, + "pass@2": 0.9994987696850394, + "pass@4": 0.9999244012935883, + "pass@8": 0.9999974638450262, + "pass@16": 0.9999999970676888, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941875, + "pass@2": 0.996166092519685, + "pass@4": 0.997643989782527, + "pass@8": 0.9989351580160544, + "pass@16": 0.9997687565508013, + "pass@32": 0.9999902507610292, + "pass@64": 0.9999999967924652, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960390625, + "pass@2": 0.997341781496063, + "pass@4": 0.9980309231346081, + "pass@8": 0.9984561726008698, + "pass@16": 0.9987499521604937, + "pass@32": 0.998950274928229, + "pass@64": 0.9989993322645678, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.964, + "2": 1.0, + "3": 1.0, + "4": 0.9996796875, + "5": 0.9966875, + "6": 0.9992578125, + "7": 0.9984375, + "8": 0.9941875, + "9": 0.9960390625 + }, + "per_op_avg_response_len": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377067, + "answer_accuracy": 0.9948787360688957, + "avg_response_len": 190.68579818895643, + "resp_tokens_sum": 72271443, + "pass_at_k": { + "pass@1": 0.9948787360688957, + "pass@2": 0.9965863642044744, + "pass@4": 0.9976738723895583, + "pass@8": 0.9983806804106521, + "pass@16": 0.9988297281125164, + "pass@32": 0.9990484158872104, + "pass@64": 0.9991555623541603, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9659845132743363, + "pass@2": 0.9762676236266927, + "pass@4": 0.9840518931261909, + "pass@8": 0.9893819037909549, + "pass@16": 0.992827041455884, + "pass@32": 0.9946400636162834, + "pass@64": 0.9955741007118146, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989990234375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999955834622234, + "pass@8": 0.9999999965852078, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962053571428572, + "pass@2": 0.9975288245219347, + "pass@4": 0.998544181679671, + "pass@8": 0.9993806306839305, + "pass@16": 0.9998935171160672, + "pass@32": 0.9999980250035232, + "pass@64": 0.9999999999662323, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9993728305688859, + "pass@4": 0.9999376890018925, + "pass@8": 0.9999995021997173, + "pass@16": 0.9999999999883278, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9989983974358975, + "pass@2": 0.9999191619977791, + "pass@4": 0.9999993645626026, + "pass@8": 0.9999999999826888, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996875, + "pass@2": 0.9971789397804147, + "pass@4": 0.9971830980634462, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388497, + "answer_accuracy": 0.9944733985910879, + "avg_response_len": 198.1560528956422, + "resp_tokens_sum": 77410851, + "pass_at_k": { + "pass@1": 0.9944733985910879, + "pass@2": 0.9964733658579378, + "pass@4": 0.9978873808860393, + "pass@8": 0.9987639465682274, + "pass@16": 0.9992391331902407, + "pass@32": 0.9995444570264099, + "pass@64": 0.9997914355526902, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9602664877300614, + "pass@2": 0.973362926307908, + "pass@4": 0.9839911745119283, + "pass@8": 0.9905991426701357, + "pass@16": 0.9937449653260887, + "pass@32": 0.9958896624995128, + "pass@64": 0.9980494756223309, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999803862832364, + "pass@4": 0.9999999103372947, + "pass@8": 0.9999999999996655, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987464183381088, + "pass@2": 0.9995321982943393, + "pass@4": 0.9999272856831292, + "pass@8": 0.9999984918054343, + "pass@16": 0.9999999996818834, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9992322976878613, + "pass@2": 0.9998545668153475, + "pass@4": 0.9999935653816394, + "pass@8": 0.9999999913233404, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953125, + "pass@2": 0.9974374792059445, + "pass@4": 0.9986580279577728, + "pass@8": 0.999449156942593, + "pass@16": 0.9999055148797769, + "pass@32": 0.9999982475383377, + "pass@64": 0.999999999970037, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9971570147107359, + "pass@4": 0.9978299348996046, + "pass@8": 0.9984680331236191, + "pass@16": 0.9992530482904565, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379817, + "answer_accuracy": 0.9934115542350184, + "avg_response_len": 211.65934413709408, + "resp_tokens_sum": 80924987, + "pass_at_k": { + "pass@1": 0.9934115542350184, + "pass@2": 0.995576261898674, + "pass@4": 0.9971791349252583, + "pass@8": 0.9982914141961841, + "pass@16": 0.998971215721579, + "pass@32": 0.9993181717967106, + "pass@64": 0.9994921540740299, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965625, + "pass@2": 0.9748288576801035, + "pass@4": 0.9819707584126611, + "pass@8": 0.9875054318482567, + "pass@16": 0.9914189594286545, + "pass@32": 0.9939477801119064, + "pass@64": 0.9954718430329116, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940716911764705, + "pass@2": 0.9974872626215842, + "pass@4": 0.9994001792055407, + "pass@8": 0.9999587340911859, + "pass@16": 0.9999998237655816, + "pass@32": 0.9999999999993819, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990881309148265, + "pass@2": 0.9997415174246752, + "pass@4": 0.9999804702054199, + "pass@8": 0.9999999148131528, + "pass@16": 0.999999999999565, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998046875, + "pass@2": 0.999243901375461, + "pass@4": 0.999834458192726, + "pass@8": 0.9999925161507783, + "pass@16": 0.9999999907330216, + "pass@32": 0.9999999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9884806681681682, + "pass@2": 0.9912943208956989, + "pass@4": 0.9943561204624196, + "pass@8": 0.9973895114319368, + "pass@16": 0.99940630260805, + "pass@32": 0.999972591246004, + "pass@64": 0.9999999903997061, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954782958199357, + "pass@2": 0.9977260931209967, + "pass@4": 0.9992145500502148, + "pass@8": 0.9998966223073351, + "pass@16": 0.999998180165534, + "pass@32": 0.9999999997896366, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.58941953125, + "avg_response_len": 358.49741640625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.58941953125, + "pass@2": 0.6611463582677087, + "pass@4": 0.7206630594394348, + "pass@8": 0.768370901754655, + "pass@16": 0.8051262293835766, + "pass@32": 0.8335568122338011, + "pass@64": 0.8565503150232208, + "pass@128": 0.8755 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9586171875, + "pass@2": 0.9686478838582683, + "pass@4": 0.9755326624484442, + "pass@8": 0.9800651950127534, + "pass@16": 0.9828574427616348, + "pass@32": 0.9847804539997522, + "pass@64": 0.9862161770242505, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.9145390625, + "pass@2": 0.9336103592519694, + "pass@4": 0.9449217005999251, + "pass@8": 0.9527443869458525, + "pass@16": 0.959081585245157, + "pass@32": 0.9655409489733907, + "pass@64": 0.9719472610563868, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.8276640625, + "pass@2": 0.8708352608267721, + "pass@4": 0.8974497278777658, + "pass@8": 0.9151390713116231, + "pass@16": 0.9282975025934377, + "pass@32": 0.9392109882089544, + "pass@64": 0.9478419183711728, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.812375, + "pass@2": 0.8693993602362209, + "pass@4": 0.9015568980127484, + "pass@8": 0.9207861570009728, + "pass@16": 0.9336252783347806, + "pass@32": 0.9440661346296119, + "pass@64": 0.9530938332686565, + "pass@128": 0.96 + }, + "15": { + "pass@1": 0.70603125, + "pass@2": 0.787097317913386, + "pass@4": 0.8406025208098982, + "pass@8": 0.87726077892471, + "pass@16": 0.9027721805519037, + "pass@32": 0.9205600648585205, + "pass@64": 0.9333755818881951, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.5262265625, + "pass@2": 0.6398211122047237, + "pass@4": 0.7261620204349456, + "pass@8": 0.7884667385639216, + "pass@16": 0.8334924020394414, + "pass@32": 0.8677150847374071, + "pass@64": 0.8928507015950615, + "pass@128": 0.909 + }, + "17": { + "pass@1": 0.4210078125, + "pass@2": 0.5368409202755909, + "pass@4": 0.6346390308398953, + "pass@8": 0.7096397214779312, + "pass@16": 0.762894409747831, + "pass@32": 0.800335830526573, + "pass@64": 0.8295895976104661, + "pass@128": 0.854 + }, + "18": { + "pass@1": 0.324109375, + "pass@2": 0.42581311515748055, + "pass@4": 0.5186248741094855, + "pass@8": 0.5981313316537631, + "pass@16": 0.6631307957628935, + "pass@32": 0.7148777344144958, + "pass@64": 0.7562454779343195, + "pass@128": 0.791 + }, + "19": { + "pass@1": 0.2183984375, + "pass@2": 0.31187549212598403, + "pass@4": 0.41036086520434906, + "pass@8": 0.5003046941579835, + "pass@16": 0.5738219700180764, + "pass@32": 0.6324042341203246, + "pass@64": 0.6825709398695216, + "pass@128": 0.728 + }, + "20": { + "pass@1": 0.1852265625, + "pass@2": 0.2675227608267715, + "pass@4": 0.3567802940569926, + "pass@8": 0.4411709424970763, + "pass@16": 0.5112887267804792, + "pass@32": 0.5660766478689608, + "pass@64": 0.6117716616140736, + "pass@128": 0.652 + } + }, + "per_op_accuracy": { + "11": 0.9586171875, + "12": 0.9145390625, + "13": 0.8276640625, + "14": 0.812375, + "15": 0.70603125, + "16": 0.5262265625, + "17": 0.4210078125, + "18": 0.324109375, + "19": 0.2183984375, + "20": 0.1852265625 + }, + "per_op_avg_response_len": { + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 254673, + "answer_accuracy": 0.5912727526002972, + "avg_response_len": 340.9688916233284, + "resp_tokens_sum": 146862121, + "pass_at_k": { + "pass@1": 0.5912727526002972, + "pass@2": 0.6622820532695326, + "pass@4": 0.7210452750891307, + "pass@8": 0.7676588819671737, + "pass@16": 0.8029748732284443, + "pass@32": 0.830823943045339, + "pass@64": 0.8545745693511848, + "pass@128": 0.874591381872214 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9477912535410765, + "pass@2": 0.9614141163480627, + "pass@4": 0.971306078861247, + "pass@8": 0.9777222153354097, + "pass@16": 0.9820192850495909, + "pass@32": 0.9859081102502127, + "pass@64": 0.9893626160626768, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.9022775423728814, + "pass@2": 0.9216494422572181, + "pass@4": 0.9329050222006143, + "pass@8": 0.9403578890122049, + "pass@16": 0.9464571379224949, + "pass@32": 0.9536009096454778, + "pass@64": 0.9612807038635403, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.8387539184952978, + "pass@2": 0.883299805000864, + "pass@4": 0.9113433348260935, + "pass@8": 0.9295898515647626, + "pass@16": 0.943287279603375, + "pass@32": 0.9551149405367227, + "pass@64": 0.9642286009171162, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8147390965732088, + "pass@2": 0.8667948340569573, + "pass@4": 0.8942816747789692, + "pass@8": 0.9112592074701689, + "pass@16": 0.9250196885538422, + "pass@32": 0.9384068954202733, + "pass@64": 0.9499602716149194, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.7185016257225434, + "pass@2": 0.7971889792226117, + "pass@4": 0.847934874108944, + "pass@8": 0.8830414368074523, + "pass@16": 0.9085139235206722, + "pass@32": 0.9277406485745909, + "pass@64": 0.9418027604093345, + "pass@128": 0.953757225433526 + }, + "16": { + "pass@1": 0.5365171370967742, + "pass@2": 0.6455339566929132, + "pass@4": 0.7244478673028772, + "pass@8": 0.779777220522581, + "pass@16": 0.8186884395276981, + "pass@32": 0.8485246873326461, + "pass@64": 0.8727635997646052, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.420497651734104, + "pass@2": 0.5373062787310544, + "pass@4": 0.6400003012629196, + "pass@8": 0.7204827202394027, + "pass@16": 0.7750126815064169, + "pass@32": 0.8111233885916674, + "pass@64": 0.8380741227154171, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.30125, + "pass@2": 0.404013990438695, + "pass@4": 0.4990766556858963, + "pass@8": 0.5797528146962918, + "pass@16": 0.6458841179962546, + "pass@32": 0.7010997597155946, + "pass@64": 0.7502144482870602, + "pass@128": 0.7971428571428572 + }, + "19": { + "pass@1": 0.21410472972972974, + "pass@2": 0.30520899836844734, + "pass@4": 0.40122690744738015, + "pass@8": 0.4897382851155873, + "pass@16": 0.5619964012938833, + "pass@32": 0.6181259558492432, + "pass@64": 0.6659910429721136, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20845063813813813, + "pass@2": 0.29438998545789885, + "pass@4": 0.38355911817329136, + "pass@8": 0.46110962180572834, + "pass@16": 0.5194414030776506, + "pass@32": 0.5654700745427053, + "pass@64": 0.6091131851339564, + "pass@128": 0.6516516516516516 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 257305, + "answer_accuracy": 0.5977387191495689, + "avg_response_len": 354.56358255278025, + "resp_tokens_sum": 152626858, + "pass_at_k": { + "pass@1": 0.5977387191495689, + "pass@2": 0.6694296255452482, + "pass@4": 0.7285026617714294, + "pass@8": 0.7757530927649476, + "pass@16": 0.8129403284610648, + "pass@32": 0.8421728400089094, + "pass@64": 0.8656627117097457, + "pass@128": 0.8846268212905144 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9688911897590361, + "pass@2": 0.9749675374727257, + "pass@4": 0.9774471477021697, + "pass@8": 0.9784483556651219, + "pass@16": 0.9788544566955297, + "pass@32": 0.978915040933753, + "pass@64": 0.9789156626487805, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.9082827194357367, + "pass@2": 0.9269358755214379, + "pass@4": 0.9375423858655597, + "pass@8": 0.9452183170723136, + "pass@16": 0.9520156669843709, + "pass@32": 0.958861937474049, + "pass@64": 0.9660584025362567, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8396358543417367, + "pass@2": 0.8762106712763842, + "pass@4": 0.8982317819516257, + "pass@8": 0.9130115564102529, + "pass@16": 0.9240476363783747, + "pass@32": 0.9335026128044098, + "pass@64": 0.9420415640994138, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8277366593567251, + "pass@2": 0.8849975681493761, + "pass@4": 0.9186248646002576, + "pass@8": 0.939483467623284, + "pass@16": 0.953015240904691, + "pass@32": 0.9622001675120875, + "pass@64": 0.968673485702234, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7206202651515151, + "pass@2": 0.8023443092340725, + "pass@4": 0.8544347351467427, + "pass@8": 0.8898100023410217, + "pass@16": 0.9154028263055654, + "pass@32": 0.9328111580456316, + "pass@64": 0.9438559391462911, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5157140313390314, + "pass@2": 0.6320804450725708, + "pass@4": 0.7233764438633206, + "pass@8": 0.7903596603735945, + "pass@16": 0.8410317719344832, + "pass@32": 0.8811910225527382, + "pass@64": 0.9109104486196058, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.45545419254658387, + "pass@2": 0.5785563744070035, + "pass@4": 0.6760754613553737, + "pass@8": 0.7431020871647102, + "pass@16": 0.7875528862860809, + "pass@32": 0.8186175463395166, + "pass@64": 0.8449973543887125, + "pass@128": 0.8695652173913043 + }, + "18": { + "pass@1": 0.3417279411764706, + "pass@2": 0.44516594777674856, + "pass@4": 0.5401025474756829, + "pass@8": 0.6218323060002307, + "pass@16": 0.6893669338501651, + "pass@32": 0.7441713183922153, + "pass@64": 0.7855624371543468, + "pass@128": 0.8147058823529412 + }, + "19": { + "pass@1": 0.23056175595238096, + "pass@2": 0.3243000386670416, + "pass@4": 0.42189163491840287, + "pass@8": 0.5118960174363878, + "pass@16": 0.5878588842096898, + "pass@32": 0.6511562770502672, + "pass@64": 0.7078413277014779, + "pass@128": 0.7559523809523809 + }, + "20": { + "pass@1": 0.17009730538922155, + "pass@2": 0.2490522154509878, + "pass@4": 0.3370424524965817, + "pass@8": 0.42361443906161345, + "pass@16": 0.4989430300289183, + "pass@32": 0.5583541344634607, + "pass@64": 0.6055054141936476, + "pass@128": 0.6467065868263473 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 242479, + "answer_accuracy": 0.5789630768643031, + "avg_response_len": 380.5673947509169, + "resp_tokens_sum": 159387714, + "pass_at_k": { + "pass@1": 0.5789630768643031, + "pass@2": 0.651464744106524, + "pass@4": 0.7122123447989714, + "pass@8": 0.7615156567110603, + "pass@16": 0.7993073107602144, + "pass@32": 0.8275117032519711, + "pass@64": 0.8492163890236676, + "pass@128": 0.8670537897310513 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.959920634920635, + "pass@2": 0.9700935820522439, + "pass@4": 0.9782513129311214, + "pass@8": 0.9843949172016934, + "pass@16": 0.9880157317975031, + "pass@32": 0.9896987221949869, + "pass@64": 0.9903846779515253, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9339162844036697, + "pass@2": 0.953070068506345, + "pass@4": 0.9651293628434054, + "pass@8": 0.9734955690809288, + "pass@16": 0.9796414698855651, + "pass@32": 0.9849824737145875, + "pass@64": 0.9892393317419794, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8035542052469136, + "pass@2": 0.8526401647710703, + "pass@4": 0.8829088207145402, + "pass@8": 0.903255503160504, + "pass@16": 0.9182217722619813, + "pass@32": 0.9298422512548313, + "pass@64": 0.93809924782445, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7945335682492581, + "pass@2": 0.856050594055936, + "pass@4": 0.8911655095412775, + "pass@8": 0.910886069661382, + "pass@16": 0.9221446466456787, + "pass@32": 0.9310536021085168, + "pass@64": 0.9402678159350233, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6778549382716049, + "pass@2": 0.7607910773306114, + "pass@4": 0.818683925215366, + "pass@8": 0.8583059907925742, + "pass@16": 0.8837760195460365, + "pass@32": 0.9004139453600427, + "pass@64": 0.9137017495934854, + "pass@128": 0.9259259259259259 + }, + "16": { + "pass@1": 0.5277009587020649, + "pass@2": 0.6426116501753651, + "pass@4": 0.7306137161505246, + "pass@8": 0.7944529776129478, + "pass@16": 0.8392236986337849, + "pass@32": 0.8713108635642363, + "pass@64": 0.8925204076771432, + "pass@128": 0.9056047197640118 + }, + "17": { + "pass@1": 0.3881306475903614, + "pass@2": 0.4958969737216582, + "pass@4": 0.5888633374894398, + "pass@8": 0.6658850247230739, + "pass@16": 0.7263493751882383, + "pass@32": 0.7713623137124465, + "pass@64": 0.8058036236077262, + "pass@128": 0.8343373493975904 + }, + "18": { + "pass@1": 0.3305947580645161, + "pass@2": 0.4291993427736854, + "pass@4": 0.517139285411904, + "pass@8": 0.5928866531289754, + "pass@16": 0.6538277321133854, + "pass@32": 0.6983050653570473, + "pass@64": 0.7309006206495801, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.21037103474320243, + "pass@2": 0.30597003829959324, + "pass@4": 0.40784506251295616, + "pass@8": 0.49916852977603016, + "pass@16": 0.5714700401593907, + "pass@32": 0.6277334795578147, + "pass@64": 0.6735989016680082, + "pass@128": 0.716012084592145 + }, + "20": { + "pass@1": 0.17717717717717718, + "pass@2": 0.25918154867465903, + "pass@4": 0.3497985842985841, + "pass@8": 0.43884148885642665, + "pass@16": 0.5155188213992907, + "pass@32": 0.5744289253316636, + "pass@64": 0.6207152030744365, + "pass@128": 0.6576576576576577 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7811833881578948, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7811833881578948, + "pass@2": 0.8198620428408456, + "pass@4": 0.8518352259487112, + "pass@8": 0.877370433230154, + "pass@16": 0.8969685249807203, + "pass@32": 0.912069660199031, + "pass@64": 0.924255226099405, + "pass@128": 0.9343157894736842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.964, + "pass@2": 0.9748387057086616, + "pass@4": 0.9833349187289088, + "pass@8": 0.9891501055647638, + "pass@16": 0.9926545771584487, + "pass@32": 0.99481551787825, + "pass@64": 0.9963468166102103, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996796875, + "pass@2": 0.9999547244094489, + "pass@4": 0.9999985867079115, + "pass@8": 0.9999999989072665, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966875, + "pass@2": 0.9983604822834645, + "pass@4": 0.999337447225347, + "pass@8": 0.9997908682563258, + "pass@16": 0.999966397971859, + "pass@32": 0.9999993778758997, + "pass@64": 0.9999999999893631, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992578125, + "pass@2": 0.9997546751968504, + "pass@4": 0.9999684317585302, + "pass@8": 0.999999446635866, + "pass@16": 0.9999999998888393, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984375, + "pass@2": 0.9994987696850394, + "pass@4": 0.9999244012935883, + "pass@8": 0.9999974638450262, + "pass@16": 0.9999999970676888, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941875, + "pass@2": 0.996166092519685, + "pass@4": 0.997643989782527, + "pass@8": 0.9989351580160544, + "pass@16": 0.9997687565508013, + "pass@32": 0.9999902507610292, + "pass@64": 0.9999999967924652, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960390625, + "pass@2": 0.997341781496063, + "pass@4": 0.9980309231346081, + "pass@8": 0.9984561726008698, + "pass@16": 0.9987499521604937, + "pass@32": 0.998950274928229, + "pass@64": 0.9989993322645678, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9586171875, + "pass@2": 0.9686478838582683, + "pass@4": 0.9755326624484442, + "pass@8": 0.9800651950127534, + "pass@16": 0.9828574427616348, + "pass@32": 0.9847804539997522, + "pass@64": 0.9862161770242505, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.9145390625, + "pass@2": 0.9336103592519694, + "pass@4": 0.9449217005999251, + "pass@8": 0.9527443869458525, + "pass@16": 0.959081585245157, + "pass@32": 0.9655409489733907, + "pass@64": 0.9719472610563868, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.8276640625, + "pass@2": 0.8708352608267721, + "pass@4": 0.8974497278777658, + "pass@8": 0.9151390713116231, + "pass@16": 0.9282975025934377, + "pass@32": 0.9392109882089544, + "pass@64": 0.9478419183711728, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.812375, + "pass@2": 0.8693993602362209, + "pass@4": 0.9015568980127484, + "pass@8": 0.9207861570009728, + "pass@16": 0.9336252783347806, + "pass@32": 0.9440661346296119, + "pass@64": 0.9530938332686565, + "pass@128": 0.96 + }, + "15": { + "pass@1": 0.70603125, + "pass@2": 0.787097317913386, + "pass@4": 0.8406025208098982, + "pass@8": 0.87726077892471, + "pass@16": 0.9027721805519037, + "pass@32": 0.9205600648585205, + "pass@64": 0.9333755818881951, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.5262265625, + "pass@2": 0.6398211122047237, + "pass@4": 0.7261620204349456, + "pass@8": 0.7884667385639216, + "pass@16": 0.8334924020394414, + "pass@32": 0.8677150847374071, + "pass@64": 0.8928507015950615, + "pass@128": 0.909 + }, + "17": { + "pass@1": 0.4210078125, + "pass@2": 0.5368409202755909, + "pass@4": 0.6346390308398953, + "pass@8": 0.7096397214779312, + "pass@16": 0.762894409747831, + "pass@32": 0.800335830526573, + "pass@64": 0.8295895976104661, + "pass@128": 0.854 + }, + "18": { + "pass@1": 0.324109375, + "pass@2": 0.42581311515748055, + "pass@4": 0.5186248741094855, + "pass@8": 0.5981313316537631, + "pass@16": 0.6631307957628935, + "pass@32": 0.7148777344144958, + "pass@64": 0.7562454779343195, + "pass@128": 0.791 + }, + "19": { + "pass@1": 0.2183984375, + "pass@2": 0.31187549212598403, + "pass@4": 0.41036086520434906, + "pass@8": 0.5003046941579835, + "pass@16": 0.5738219700180764, + "pass@32": 0.6324042341203246, + "pass@64": 0.6825709398695216, + "pass@128": 0.728 + }, + "20": { + "pass@1": 0.1852265625, + "pass@2": 0.2675227608267715, + "pass@4": 0.3567802940569926, + "pass@8": 0.4411709424970763, + "pass@16": 0.5112887267804792, + "pass@32": 0.5660766478689608, + "pass@64": 0.6117716616140736, + "pass@128": 0.652 + } + }, + "per_op_accuracy": { + "10": 0.964, + "2": 1.0, + "3": 1.0, + "4": 0.9996796875, + "5": 0.9966875, + "6": 0.9992578125, + "7": 0.9984375, + "8": 0.9941875, + "9": 0.9960390625, + "11": 0.9586171875, + "12": 0.9145390625, + "13": 0.8276640625, + "14": 0.812375, + "15": 0.70603125, + "16": 0.5262265625, + "17": 0.4210078125, + "18": 0.324109375, + "19": 0.2183984375, + "20": 0.1852265625 + }, + "per_op_avg_response_len": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125, + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125, + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 631740, + "answer_accuracy": 0.7801879149541574, + "avg_response_len": 270.626141123933, + "resp_tokens_sum": 219133564, + "pass_at_k": { + "pass@1": 0.7801879149541574, + "pass@2": 0.8187593002942418, + "pass@4": 0.8505263494815603, + "pass@8": 0.8756524395376909, + "pass@16": 0.8946483201635945, + "pass@32": 0.9095644843170411, + "pass@64": 0.9222483474545463, + "pass@128": 0.9329750237116662 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9659845132743363, + "pass@2": 0.9762676236266927, + "pass@4": 0.9840518931261909, + "pass@8": 0.9893819037909549, + "pass@16": 0.992827041455884, + "pass@32": 0.9946400636162834, + "pass@64": 0.9955741007118146, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989990234375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999955834622234, + "pass@8": 0.9999999965852078, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962053571428572, + "pass@2": 0.9975288245219347, + "pass@4": 0.998544181679671, + "pass@8": 0.9993806306839305, + "pass@16": 0.9998935171160672, + "pass@32": 0.9999980250035232, + "pass@64": 0.9999999999662323, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9993728305688859, + "pass@4": 0.9999376890018925, + "pass@8": 0.9999995021997173, + "pass@16": 0.9999999999883278, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9989983974358975, + "pass@2": 0.9999191619977791, + "pass@4": 0.9999993645626026, + "pass@8": 0.9999999999826888, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996875, + "pass@2": 0.9971789397804147, + "pass@4": 0.9971830980634462, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9477912535410765, + "pass@2": 0.9614141163480627, + "pass@4": 0.971306078861247, + "pass@8": 0.9777222153354097, + "pass@16": 0.9820192850495909, + "pass@32": 0.9859081102502127, + "pass@64": 0.9893626160626768, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.9022775423728814, + "pass@2": 0.9216494422572181, + "pass@4": 0.9329050222006143, + "pass@8": 0.9403578890122049, + "pass@16": 0.9464571379224949, + "pass@32": 0.9536009096454778, + "pass@64": 0.9612807038635403, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.8387539184952978, + "pass@2": 0.883299805000864, + "pass@4": 0.9113433348260935, + "pass@8": 0.9295898515647626, + "pass@16": 0.943287279603375, + "pass@32": 0.9551149405367227, + "pass@64": 0.9642286009171162, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8147390965732088, + "pass@2": 0.8667948340569573, + "pass@4": 0.8942816747789692, + "pass@8": 0.9112592074701689, + "pass@16": 0.9250196885538422, + "pass@32": 0.9384068954202733, + "pass@64": 0.9499602716149194, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.7185016257225434, + "pass@2": 0.7971889792226117, + "pass@4": 0.847934874108944, + "pass@8": 0.8830414368074523, + "pass@16": 0.9085139235206722, + "pass@32": 0.9277406485745909, + "pass@64": 0.9418027604093345, + "pass@128": 0.953757225433526 + }, + "16": { + "pass@1": 0.5365171370967742, + "pass@2": 0.6455339566929132, + "pass@4": 0.7244478673028772, + "pass@8": 0.779777220522581, + "pass@16": 0.8186884395276981, + "pass@32": 0.8485246873326461, + "pass@64": 0.8727635997646052, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.420497651734104, + "pass@2": 0.5373062787310544, + "pass@4": 0.6400003012629196, + "pass@8": 0.7204827202394027, + "pass@16": 0.7750126815064169, + "pass@32": 0.8111233885916674, + "pass@64": 0.8380741227154171, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.30125, + "pass@2": 0.404013990438695, + "pass@4": 0.4990766556858963, + "pass@8": 0.5797528146962918, + "pass@16": 0.6458841179962546, + "pass@32": 0.7010997597155946, + "pass@64": 0.7502144482870602, + "pass@128": 0.7971428571428572 + }, + "19": { + "pass@1": 0.21410472972972974, + "pass@2": 0.30520899836844734, + "pass@4": 0.40122690744738015, + "pass@8": 0.4897382851155873, + "pass@16": 0.5619964012938833, + "pass@32": 0.6181259558492432, + "pass@64": 0.6659910429721136, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20845063813813813, + "pass@2": 0.29438998545789885, + "pass@4": 0.38355911817329136, + "pass@8": 0.46110962180572834, + "pass@16": 0.5194414030776506, + "pass@32": 0.5654700745427053, + "pass@64": 0.6091131851339564, + "pass@128": 0.6516516516516516 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 645802, + "answer_accuracy": 0.786489185502728, + "avg_response_len": 280.1511459957132, + "resp_tokens_sum": 230037709, + "pass_at_k": { + "pass@1": 0.786489185502728, + "pass@2": 0.8250239350439653, + "pass@4": 0.8566651189402132, + "pass@8": 0.8818527226648039, + "pass@16": 0.9015738361825739, + "pass@32": 0.9170439507084295, + "pass@64": 0.9294757850018294, + "pass@128": 0.9395167575993765 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9602664877300614, + "pass@2": 0.973362926307908, + "pass@4": 0.9839911745119283, + "pass@8": 0.9905991426701357, + "pass@16": 0.9937449653260887, + "pass@32": 0.9958896624995128, + "pass@64": 0.9980494756223309, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999803862832364, + "pass@4": 0.9999999103372947, + "pass@8": 0.9999999999996655, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987464183381088, + "pass@2": 0.9995321982943393, + "pass@4": 0.9999272856831292, + "pass@8": 0.9999984918054343, + "pass@16": 0.9999999996818834, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9992322976878613, + "pass@2": 0.9998545668153475, + "pass@4": 0.9999935653816394, + "pass@8": 0.9999999913233404, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953125, + "pass@2": 0.9974374792059445, + "pass@4": 0.9986580279577728, + "pass@8": 0.999449156942593, + "pass@16": 0.9999055148797769, + "pass@32": 0.9999982475383377, + "pass@64": 0.999999999970037, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9971570147107359, + "pass@4": 0.9978299348996046, + "pass@8": 0.9984680331236191, + "pass@16": 0.9992530482904565, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9688911897590361, + "pass@2": 0.9749675374727257, + "pass@4": 0.9774471477021697, + "pass@8": 0.9784483556651219, + "pass@16": 0.9788544566955297, + "pass@32": 0.978915040933753, + "pass@64": 0.9789156626487805, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.9082827194357367, + "pass@2": 0.9269358755214379, + "pass@4": 0.9375423858655597, + "pass@8": 0.9452183170723136, + "pass@16": 0.9520156669843709, + "pass@32": 0.958861937474049, + "pass@64": 0.9660584025362567, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8396358543417367, + "pass@2": 0.8762106712763842, + "pass@4": 0.8982317819516257, + "pass@8": 0.9130115564102529, + "pass@16": 0.9240476363783747, + "pass@32": 0.9335026128044098, + "pass@64": 0.9420415640994138, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8277366593567251, + "pass@2": 0.8849975681493761, + "pass@4": 0.9186248646002576, + "pass@8": 0.939483467623284, + "pass@16": 0.953015240904691, + "pass@32": 0.9622001675120875, + "pass@64": 0.968673485702234, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7206202651515151, + "pass@2": 0.8023443092340725, + "pass@4": 0.8544347351467427, + "pass@8": 0.8898100023410217, + "pass@16": 0.9154028263055654, + "pass@32": 0.9328111580456316, + "pass@64": 0.9438559391462911, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5157140313390314, + "pass@2": 0.6320804450725708, + "pass@4": 0.7233764438633206, + "pass@8": 0.7903596603735945, + "pass@16": 0.8410317719344832, + "pass@32": 0.8811910225527382, + "pass@64": 0.9109104486196058, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.45545419254658387, + "pass@2": 0.5785563744070035, + "pass@4": 0.6760754613553737, + "pass@8": 0.7431020871647102, + "pass@16": 0.7875528862860809, + "pass@32": 0.8186175463395166, + "pass@64": 0.8449973543887125, + "pass@128": 0.8695652173913043 + }, + "18": { + "pass@1": 0.3417279411764706, + "pass@2": 0.44516594777674856, + "pass@4": 0.5401025474756829, + "pass@8": 0.6218323060002307, + "pass@16": 0.6893669338501651, + "pass@32": 0.7441713183922153, + "pass@64": 0.7855624371543468, + "pass@128": 0.8147058823529412 + }, + "19": { + "pass@1": 0.23056175595238096, + "pass@2": 0.3243000386670416, + "pass@4": 0.42189163491840287, + "pass@8": 0.5118960174363878, + "pass@16": 0.5878588842096898, + "pass@32": 0.6511562770502672, + "pass@64": 0.7078413277014779, + "pass@128": 0.7559523809523809 + }, + "20": { + "pass@1": 0.17009730538922155, + "pass@2": 0.2490522154509878, + "pass@4": 0.3370424524965817, + "pass@8": 0.42361443906161345, + "pass@16": 0.4989430300289183, + "pass@32": 0.5583541344634607, + "pass@64": 0.6055054141936476, + "pass@128": 0.6467065868263473 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 622296, + "answer_accuracy": 0.7767514778718645, + "avg_response_len": 299.9589353830484, + "resp_tokens_sum": 240312701, + "pass_at_k": { + "pass@1": 0.7767514778718645, + "pass@2": 0.8156860420207447, + "pass@4": 0.848207839623572, + "pass@8": 0.8745128108264176, + "pass@16": 0.8945934721469555, + "pass@32": 0.9095033826804982, + "pass@64": 0.9209329108650951, + "pass@128": 0.9303403099536667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965625, + "pass@2": 0.9748288576801035, + "pass@4": 0.9819707584126611, + "pass@8": 0.9875054318482567, + "pass@16": 0.9914189594286545, + "pass@32": 0.9939477801119064, + "pass@64": 0.9954718430329116, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940716911764705, + "pass@2": 0.9974872626215842, + "pass@4": 0.9994001792055407, + "pass@8": 0.9999587340911859, + "pass@16": 0.9999998237655816, + "pass@32": 0.9999999999993819, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990881309148265, + "pass@2": 0.9997415174246752, + "pass@4": 0.9999804702054199, + "pass@8": 0.9999999148131528, + "pass@16": 0.999999999999565, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998046875, + "pass@2": 0.999243901375461, + "pass@4": 0.999834458192726, + "pass@8": 0.9999925161507783, + "pass@16": 0.9999999907330216, + "pass@32": 0.9999999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9884806681681682, + "pass@2": 0.9912943208956989, + "pass@4": 0.9943561204624196, + "pass@8": 0.9973895114319368, + "pass@16": 0.99940630260805, + "pass@32": 0.999972591246004, + "pass@64": 0.9999999903997061, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954782958199357, + "pass@2": 0.9977260931209967, + "pass@4": 0.9992145500502148, + "pass@8": 0.9998966223073351, + "pass@16": 0.999998180165534, + "pass@32": 0.9999999997896366, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.959920634920635, + "pass@2": 0.9700935820522439, + "pass@4": 0.9782513129311214, + "pass@8": 0.9843949172016934, + "pass@16": 0.9880157317975031, + "pass@32": 0.9896987221949869, + "pass@64": 0.9903846779515253, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9339162844036697, + "pass@2": 0.953070068506345, + "pass@4": 0.9651293628434054, + "pass@8": 0.9734955690809288, + "pass@16": 0.9796414698855651, + "pass@32": 0.9849824737145875, + "pass@64": 0.9892393317419794, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8035542052469136, + "pass@2": 0.8526401647710703, + "pass@4": 0.8829088207145402, + "pass@8": 0.903255503160504, + "pass@16": 0.9182217722619813, + "pass@32": 0.9298422512548313, + "pass@64": 0.93809924782445, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7945335682492581, + "pass@2": 0.856050594055936, + "pass@4": 0.8911655095412775, + "pass@8": 0.910886069661382, + "pass@16": 0.9221446466456787, + "pass@32": 0.9310536021085168, + "pass@64": 0.9402678159350233, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6778549382716049, + "pass@2": 0.7607910773306114, + "pass@4": 0.818683925215366, + "pass@8": 0.8583059907925742, + "pass@16": 0.8837760195460365, + "pass@32": 0.9004139453600427, + "pass@64": 0.9137017495934854, + "pass@128": 0.9259259259259259 + }, + "16": { + "pass@1": 0.5277009587020649, + "pass@2": 0.6426116501753651, + "pass@4": 0.7306137161505246, + "pass@8": 0.7944529776129478, + "pass@16": 0.8392236986337849, + "pass@32": 0.8713108635642363, + "pass@64": 0.8925204076771432, + "pass@128": 0.9056047197640118 + }, + "17": { + "pass@1": 0.3881306475903614, + "pass@2": 0.4958969737216582, + "pass@4": 0.5888633374894398, + "pass@8": 0.6658850247230739, + "pass@16": 0.7263493751882383, + "pass@32": 0.7713623137124465, + "pass@64": 0.8058036236077262, + "pass@128": 0.8343373493975904 + }, + "18": { + "pass@1": 0.3305947580645161, + "pass@2": 0.4291993427736854, + "pass@4": 0.517139285411904, + "pass@8": 0.5928866531289754, + "pass@16": 0.6538277321133854, + "pass@32": 0.6983050653570473, + "pass@64": 0.7309006206495801, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.21037103474320243, + "pass@2": 0.30597003829959324, + "pass@4": 0.40784506251295616, + "pass@8": 0.49916852977603016, + "pass@16": 0.5714700401593907, + "pass@32": 0.6277334795578147, + "pass@64": 0.6735989016680082, + "pass@128": 0.716012084592145 + }, + "20": { + "pass@1": 0.17717717717717718, + "pass@2": 0.25918154867465903, + "pass@4": 0.3497985842985841, + "pass@8": 0.43884148885642665, + "pass@16": 0.5155188213992907, + "pass@32": 0.5744289253316636, + "pass@64": 0.6207152030744365, + "pass@128": 0.6576576576576577 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl b/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2d3eedecb1b75a346f7dfa064b04f71e57c66d37 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6802a675df39f389fc3a9300a876eee90a7f6cb2388ccc828a9b1e9473ab1543 +size 2656247437 diff --git a/results/composition/SDAR-100M-block4-epoch4-op9-12/summary.csv b/results/composition/SDAR-100M-block4-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..e054e206bb958c05e4a1224d56dbf479e6b9c888 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op9-12,0.9942543402777778,0.58941953125,0.7811833881578948,0.0,0.0,0.0,200.17993142361112,358.49741640625,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block4-epoch4-op9-12/summary.json b/results/composition/SDAR-100M-block4-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7247a9069949aedd1b273edf2ffcc9a5072d8a22 --- /dev/null +++ b/results/composition/SDAR-100M-block4-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op9-12", + "id_acc": 0.9942543402777778, + "ood_acc": 0.58941953125, + "total_acc": 0.7811833881578948, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.17993142361112, + "ood_avg_resp_len": 358.49741640625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 631740, + "answer_accuracy": 0.7801879149541574, + "avg_response_len": 270.626141123933, + "resp_tokens_sum": 219133564, + "pass_at_k": { + "pass@1": 0.7801879149541574, + "pass@2": 0.8187593002942418, + "pass@4": 0.8505263494815603, + "pass@8": 0.8756524395376909, + "pass@16": 0.8946483201635945, + "pass@32": 0.9095644843170411, + "pass@64": 0.9222483474545463, + "pass@128": 0.9329750237116662 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9659845132743363, + "pass@2": 0.9762676236266927, + "pass@4": 0.9840518931261909, + "pass@8": 0.9893819037909549, + "pass@16": 0.992827041455884, + "pass@32": 0.9946400636162834, + "pass@64": 0.9955741007118146, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989990234375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999955834622234, + "pass@8": 0.9999999965852078, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962053571428572, + "pass@2": 0.9975288245219347, + "pass@4": 0.998544181679671, + "pass@8": 0.9993806306839305, + "pass@16": 0.9998935171160672, + "pass@32": 0.9999980250035232, + "pass@64": 0.9999999999662323, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9993728305688859, + "pass@4": 0.9999376890018925, + "pass@8": 0.9999995021997173, + "pass@16": 0.9999999999883278, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9989983974358975, + "pass@2": 0.9999191619977791, + "pass@4": 0.9999993645626026, + "pass@8": 0.9999999999826888, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996875, + "pass@2": 0.9971789397804147, + "pass@4": 0.9971830980634462, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9477912535410765, + "pass@2": 0.9614141163480627, + "pass@4": 0.971306078861247, + "pass@8": 0.9777222153354097, + "pass@16": 0.9820192850495909, + "pass@32": 0.9859081102502127, + "pass@64": 0.9893626160626768, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.9022775423728814, + "pass@2": 0.9216494422572181, + "pass@4": 0.9329050222006143, + "pass@8": 0.9403578890122049, + "pass@16": 0.9464571379224949, + "pass@32": 0.9536009096454778, + "pass@64": 0.9612807038635403, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.8387539184952978, + "pass@2": 0.883299805000864, + "pass@4": 0.9113433348260935, + "pass@8": 0.9295898515647626, + "pass@16": 0.943287279603375, + "pass@32": 0.9551149405367227, + "pass@64": 0.9642286009171162, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8147390965732088, + "pass@2": 0.8667948340569573, + "pass@4": 0.8942816747789692, + "pass@8": 0.9112592074701689, + "pass@16": 0.9250196885538422, + "pass@32": 0.9384068954202733, + "pass@64": 0.9499602716149194, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.7185016257225434, + "pass@2": 0.7971889792226117, + "pass@4": 0.847934874108944, + "pass@8": 0.8830414368074523, + "pass@16": 0.9085139235206722, + "pass@32": 0.9277406485745909, + "pass@64": 0.9418027604093345, + "pass@128": 0.953757225433526 + }, + "16": { + "pass@1": 0.5365171370967742, + "pass@2": 0.6455339566929132, + "pass@4": 0.7244478673028772, + "pass@8": 0.779777220522581, + "pass@16": 0.8186884395276981, + "pass@32": 0.8485246873326461, + "pass@64": 0.8727635997646052, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.420497651734104, + "pass@2": 0.5373062787310544, + "pass@4": 0.6400003012629196, + "pass@8": 0.7204827202394027, + "pass@16": 0.7750126815064169, + "pass@32": 0.8111233885916674, + "pass@64": 0.8380741227154171, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.30125, + "pass@2": 0.404013990438695, + "pass@4": 0.4990766556858963, + "pass@8": 0.5797528146962918, + "pass@16": 0.6458841179962546, + "pass@32": 0.7010997597155946, + "pass@64": 0.7502144482870602, + "pass@128": 0.7971428571428572 + }, + "19": { + "pass@1": 0.21410472972972974, + "pass@2": 0.30520899836844734, + "pass@4": 0.40122690744738015, + "pass@8": 0.4897382851155873, + "pass@16": 0.5619964012938833, + "pass@32": 0.6181259558492432, + "pass@64": 0.6659910429721136, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20845063813813813, + "pass@2": 0.29438998545789885, + "pass@4": 0.38355911817329136, + "pass@8": 0.46110962180572834, + "pass@16": 0.5194414030776506, + "pass@32": 0.5654700745427053, + "pass@64": 0.6091131851339564, + "pass@128": 0.6516516516516516 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 645802, + "answer_accuracy": 0.786489185502728, + "avg_response_len": 280.1511459957132, + "resp_tokens_sum": 230037709, + "pass_at_k": { + "pass@1": 0.786489185502728, + "pass@2": 0.8250239350439653, + "pass@4": 0.8566651189402132, + "pass@8": 0.8818527226648039, + "pass@16": 0.9015738361825739, + "pass@32": 0.9170439507084295, + "pass@64": 0.9294757850018294, + "pass@128": 0.9395167575993765 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9602664877300614, + "pass@2": 0.973362926307908, + "pass@4": 0.9839911745119283, + "pass@8": 0.9905991426701357, + "pass@16": 0.9937449653260887, + "pass@32": 0.9958896624995128, + "pass@64": 0.9980494756223309, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999803862832364, + "pass@4": 0.9999999103372947, + "pass@8": 0.9999999999996655, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987464183381088, + "pass@2": 0.9995321982943393, + "pass@4": 0.9999272856831292, + "pass@8": 0.9999984918054343, + "pass@16": 0.9999999996818834, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9992322976878613, + "pass@2": 0.9998545668153475, + "pass@4": 0.9999935653816394, + "pass@8": 0.9999999913233404, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953125, + "pass@2": 0.9974374792059445, + "pass@4": 0.9986580279577728, + "pass@8": 0.999449156942593, + "pass@16": 0.9999055148797769, + "pass@32": 0.9999982475383377, + "pass@64": 0.999999999970037, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9971570147107359, + "pass@4": 0.9978299348996046, + "pass@8": 0.9984680331236191, + "pass@16": 0.9992530482904565, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9688911897590361, + "pass@2": 0.9749675374727257, + "pass@4": 0.9774471477021697, + "pass@8": 0.9784483556651219, + "pass@16": 0.9788544566955297, + "pass@32": 0.978915040933753, + "pass@64": 0.9789156626487805, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.9082827194357367, + "pass@2": 0.9269358755214379, + "pass@4": 0.9375423858655597, + "pass@8": 0.9452183170723136, + "pass@16": 0.9520156669843709, + "pass@32": 0.958861937474049, + "pass@64": 0.9660584025362567, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8396358543417367, + "pass@2": 0.8762106712763842, + "pass@4": 0.8982317819516257, + "pass@8": 0.9130115564102529, + "pass@16": 0.9240476363783747, + "pass@32": 0.9335026128044098, + "pass@64": 0.9420415640994138, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8277366593567251, + "pass@2": 0.8849975681493761, + "pass@4": 0.9186248646002576, + "pass@8": 0.939483467623284, + "pass@16": 0.953015240904691, + "pass@32": 0.9622001675120875, + "pass@64": 0.968673485702234, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7206202651515151, + "pass@2": 0.8023443092340725, + "pass@4": 0.8544347351467427, + "pass@8": 0.8898100023410217, + "pass@16": 0.9154028263055654, + "pass@32": 0.9328111580456316, + "pass@64": 0.9438559391462911, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5157140313390314, + "pass@2": 0.6320804450725708, + "pass@4": 0.7233764438633206, + "pass@8": 0.7903596603735945, + "pass@16": 0.8410317719344832, + "pass@32": 0.8811910225527382, + "pass@64": 0.9109104486196058, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.45545419254658387, + "pass@2": 0.5785563744070035, + "pass@4": 0.6760754613553737, + "pass@8": 0.7431020871647102, + "pass@16": 0.7875528862860809, + "pass@32": 0.8186175463395166, + "pass@64": 0.8449973543887125, + "pass@128": 0.8695652173913043 + }, + "18": { + "pass@1": 0.3417279411764706, + "pass@2": 0.44516594777674856, + "pass@4": 0.5401025474756829, + "pass@8": 0.6218323060002307, + "pass@16": 0.6893669338501651, + "pass@32": 0.7441713183922153, + "pass@64": 0.7855624371543468, + "pass@128": 0.8147058823529412 + }, + "19": { + "pass@1": 0.23056175595238096, + "pass@2": 0.3243000386670416, + "pass@4": 0.42189163491840287, + "pass@8": 0.5118960174363878, + "pass@16": 0.5878588842096898, + "pass@32": 0.6511562770502672, + "pass@64": 0.7078413277014779, + "pass@128": 0.7559523809523809 + }, + "20": { + "pass@1": 0.17009730538922155, + "pass@2": 0.2490522154509878, + "pass@4": 0.3370424524965817, + "pass@8": 0.42361443906161345, + "pass@16": 0.4989430300289183, + "pass@32": 0.5583541344634607, + "pass@64": 0.6055054141936476, + "pass@128": 0.6467065868263473 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 622296, + "answer_accuracy": 0.7767514778718645, + "avg_response_len": 299.9589353830484, + "resp_tokens_sum": 240312701, + "pass_at_k": { + "pass@1": 0.7767514778718645, + "pass@2": 0.8156860420207447, + "pass@4": 0.848207839623572, + "pass@8": 0.8745128108264176, + "pass@16": 0.8945934721469555, + "pass@32": 0.9095033826804982, + "pass@64": 0.9209329108650951, + "pass@128": 0.9303403099536667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965625, + "pass@2": 0.9748288576801035, + "pass@4": 0.9819707584126611, + "pass@8": 0.9875054318482567, + "pass@16": 0.9914189594286545, + "pass@32": 0.9939477801119064, + "pass@64": 0.9954718430329116, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940716911764705, + "pass@2": 0.9974872626215842, + "pass@4": 0.9994001792055407, + "pass@8": 0.9999587340911859, + "pass@16": 0.9999998237655816, + "pass@32": 0.9999999999993819, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990881309148265, + "pass@2": 0.9997415174246752, + "pass@4": 0.9999804702054199, + "pass@8": 0.9999999148131528, + "pass@16": 0.999999999999565, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998046875, + "pass@2": 0.999243901375461, + "pass@4": 0.999834458192726, + "pass@8": 0.9999925161507783, + "pass@16": 0.9999999907330216, + "pass@32": 0.9999999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9884806681681682, + "pass@2": 0.9912943208956989, + "pass@4": 0.9943561204624196, + "pass@8": 0.9973895114319368, + "pass@16": 0.99940630260805, + "pass@32": 0.999972591246004, + "pass@64": 0.9999999903997061, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954782958199357, + "pass@2": 0.9977260931209967, + "pass@4": 0.9992145500502148, + "pass@8": 0.9998966223073351, + "pass@16": 0.999998180165534, + "pass@32": 0.9999999997896366, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.959920634920635, + "pass@2": 0.9700935820522439, + "pass@4": 0.9782513129311214, + "pass@8": 0.9843949172016934, + "pass@16": 0.9880157317975031, + "pass@32": 0.9896987221949869, + "pass@64": 0.9903846779515253, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9339162844036697, + "pass@2": 0.953070068506345, + "pass@4": 0.9651293628434054, + "pass@8": 0.9734955690809288, + "pass@16": 0.9796414698855651, + "pass@32": 0.9849824737145875, + "pass@64": 0.9892393317419794, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8035542052469136, + "pass@2": 0.8526401647710703, + "pass@4": 0.8829088207145402, + "pass@8": 0.903255503160504, + "pass@16": 0.9182217722619813, + "pass@32": 0.9298422512548313, + "pass@64": 0.93809924782445, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7945335682492581, + "pass@2": 0.856050594055936, + "pass@4": 0.8911655095412775, + "pass@8": 0.910886069661382, + "pass@16": 0.9221446466456787, + "pass@32": 0.9310536021085168, + "pass@64": 0.9402678159350233, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6778549382716049, + "pass@2": 0.7607910773306114, + "pass@4": 0.818683925215366, + "pass@8": 0.8583059907925742, + "pass@16": 0.8837760195460365, + "pass@32": 0.9004139453600427, + "pass@64": 0.9137017495934854, + "pass@128": 0.9259259259259259 + }, + "16": { + "pass@1": 0.5277009587020649, + "pass@2": 0.6426116501753651, + "pass@4": 0.7306137161505246, + "pass@8": 0.7944529776129478, + "pass@16": 0.8392236986337849, + "pass@32": 0.8713108635642363, + "pass@64": 0.8925204076771432, + "pass@128": 0.9056047197640118 + }, + "17": { + "pass@1": 0.3881306475903614, + "pass@2": 0.4958969737216582, + "pass@4": 0.5888633374894398, + "pass@8": 0.6658850247230739, + "pass@16": 0.7263493751882383, + "pass@32": 0.7713623137124465, + "pass@64": 0.8058036236077262, + "pass@128": 0.8343373493975904 + }, + "18": { + "pass@1": 0.3305947580645161, + "pass@2": 0.4291993427736854, + "pass@4": 0.517139285411904, + "pass@8": 0.5928866531289754, + "pass@16": 0.6538277321133854, + "pass@32": 0.6983050653570473, + "pass@64": 0.7309006206495801, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.21037103474320243, + "pass@2": 0.30597003829959324, + "pass@4": 0.40784506251295616, + "pass@8": 0.49916852977603016, + "pass@16": 0.5714700401593907, + "pass@32": 0.6277334795578147, + "pass@64": 0.6735989016680082, + "pass@128": 0.716012084592145 + }, + "20": { + "pass@1": 0.17717717717717718, + "pass@2": 0.25918154867465903, + "pass@4": 0.3497985842985841, + "pass@8": 0.43884148885642665, + "pass@16": 0.5155188213992907, + "pass@32": 0.5744289253316636, + "pass@64": 0.6207152030744365, + "pass@128": 0.6576576576576577 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..50519d3bf4127e82bcc8de19fa57d5a035b48a6b --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1a4fe83097db87463e120bf778aeb52f9781b7961ce279641b9ed33afa5b9de +size 1792133431 diff --git a/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json b/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7628996582c492ddad431ce123ed46f0f401973a --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op11-14", + "id": { + "answer_accuracy": 0.9792821180555555, + "avg_response_len": 198.730515625, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9792821180555555, + "pass@2": 0.984920699365702, + "pass@4": 0.9881914878660996, + "pass@8": 0.9904670562279972, + "pass@16": 0.9921862755436088, + "pass@32": 0.9935212775960051, + "pass@64": 0.9946401655204734, + "pass@128": 0.9954444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9203125, + "pass@2": 0.9373841043307087, + "pass@4": 0.9486343601424823, + "pass@8": 0.9568461594757247, + "pass@16": 0.9631299032632213, + "pass@32": 0.9681102653736894, + "pass@64": 0.9720156987023217, + "pass@128": 0.975 + }, + "2": { + "pass@1": 0.9981640625, + "pass@2": 0.9987837106299212, + "pass@4": 0.9990383347394075, + "pass@8": 0.999121313250048, + "pass@16": 0.9992352362042897, + "pass@32": 0.9994389763779528, + "pass@64": 0.9997519685039371, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9952421875, + "pass@2": 0.9970708661417323, + "pass@4": 0.9983012541244844, + "pass@8": 0.9991987368058538, + "pass@16": 0.9997891651186116, + "pass@32": 0.9999863538662602, + "pass@64": 0.9999999869805416, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9923984375, + "pass@2": 0.9935083661417322, + "pass@4": 0.994365091488564, + "pass@8": 0.9951879421387454, + "pass@16": 0.995978519344931, + "pass@32": 0.9968444264374606, + "pass@64": 0.9976963412230191, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97834375, + "pass@2": 0.983570989173229, + "pass@4": 0.9870388284589425, + "pass@8": 0.9898555406065586, + "pass@16": 0.9925189720292993, + "pass@32": 0.9950251327152541, + "pass@64": 0.9973750477309096, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99003125, + "pass@2": 0.9921589566929134, + "pass@4": 0.9934118787964004, + "pass@8": 0.9940810662118578, + "pass@16": 0.9944445402431988, + "pass@32": 0.9948313926560495, + "pass@64": 0.9953779527558758, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.985546875, + "pass@2": 0.9910484744094491, + "pass@4": 0.994010423415823, + "pass@8": 0.9954839757776164, + "pass@16": 0.9962401880505216, + "pass@32": 0.9966784691049091, + "pass@64": 0.9969404364588756, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9819921875, + "pass@2": 0.9892123523622052, + "pass@4": 0.9925807832770904, + "pass@8": 0.9947584005374547, + "pass@16": 0.9960946946258159, + "pass@32": 0.9967326365666029, + "pass@64": 0.9969710370484877, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9715078125, + "pass@2": 0.9815484744094494, + "pass@4": 0.9863424363517062, + "pass@8": 0.9896703712481156, + "pass@16": 0.9922452610125915, + "pass@32": 0.9940438452658681, + "pass@64": 0.9956330202802935, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9203125, + "2": 0.9981640625, + "3": 0.9952421875, + "4": 0.9923984375, + "5": 0.97834375, + "6": 0.99003125, + "7": 0.985546875, + "8": 0.9819921875, + "9": 0.9715078125 + }, + "per_op_avg_response_len": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 370805, + "answer_accuracy": 0.9783566573792638, + "avg_response_len": 189.4634757049983, + "resp_tokens_sum": 71808173, + "pass_at_k": { + "pass@1": 0.9783566573792638, + "pass@2": 0.984426398162996, + "pass@4": 0.9879441927028629, + "pass@8": 0.9903708442945975, + "pass@16": 0.9922021831471315, + "pass@32": 0.99362419389056, + "pass@64": 0.9947730546856574, + "pass@128": 0.9956095913542722 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231425147492626, + "pass@2": 0.9383285572433974, + "pass@4": 0.9473531416979957, + "pass@8": 0.9537757890092359, + "pass@16": 0.9589878410934526, + "pass@32": 0.9634827340913612, + "pass@64": 0.9671551508567272, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.999123831775701, + "pass@2": 0.999758536316138, + "pass@4": 0.9999827985870926, + "pass@8": 0.9999999340639316, + "pass@16": 0.999999999999756, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9958428899082569, + "pass@2": 0.9977124419080643, + "pass@4": 0.9987991609993705, + "pass@8": 0.9995675174167592, + "pass@16": 0.9999476070371531, + "pass@32": 0.9999995668075918, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9838134765625, + "pass@2": 0.9859390378937007, + "pass@4": 0.9883570163104611, + "pass@8": 0.9906581655190253, + "pass@16": 0.9926442820736442, + "pass@32": 0.9945717745930672, + "pass@64": 0.9963074639597295, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9704117063492064, + "pass@2": 0.9771399668791401, + "pass@4": 0.9815450669856742, + "pass@8": 0.9853588892526388, + "pass@16": 0.9894869670242535, + "pass@32": 0.9933388347853764, + "pass@64": 0.9967933357643882, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992936002994012, + "pass@2": 0.9954846704229336, + "pass@4": 0.9969230995264813, + "pass@8": 0.9975195058071472, + "pass@16": 0.9980010177318054, + "pass@32": 0.9987468819391533, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860623150887574, + "pass@2": 0.9925427625448447, + "pass@4": 0.9958278568544315, + "pass@8": 0.9969092535432367, + "pass@16": 0.9970388200006978, + "pass@32": 0.9970414196166396, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9829477163461539, + "pass@2": 0.9896030499192406, + "pass@4": 0.9923302090243528, + "pass@8": 0.9939964803271766, + "pass@16": 0.9951222806297688, + "pass@32": 0.9960540651646855, + "pass@64": 0.9967025035329475, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9732394366197183, + "pass@2": 0.9847652351114562, + "pass@4": 0.9913186990710667, + "pass@8": 0.9961890503448468, + "pass@16": 0.9991175807085956, + "pass@32": 0.9999597333809449, + "pass@64": 0.9999999878674508, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382626, + "answer_accuracy": 0.9794448312581914, + "avg_response_len": 196.59161000982962, + "resp_tokens_sum": 76799692, + "pass_at_k": { + "pass@1": 0.9794448312581914, + "pass@2": 0.9841983135493961, + "pass@4": 0.9869167429902689, + "pass@8": 0.9889238346741729, + "pass@16": 0.9904814962330848, + "pass@32": 0.9917710998181902, + "pass@64": 0.9930290745140739, + "pass@128": 0.9941022280471822 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9112586273006135, + "pass@2": 0.9263716879619345, + "pass@4": 0.9368261261245718, + "pass@8": 0.9442747604605302, + "pass@16": 0.9497976670404737, + "pass@32": 0.954829192728206, + "pass@64": 0.9594253227868381, + "pass@128": 0.9631901840490797 + }, + "2": { + "pass@1": 0.9988839285714286, + "pass@2": 0.999595753655793, + "pass@4": 0.9999491804595854, + "pass@8": 0.9999993314708793, + "pass@16": 0.9999999999510504, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.993655303030303, + "pass@2": 0.9962635707468384, + "pass@4": 0.9979543449114315, + "pass@8": 0.9990869648235982, + "pass@16": 0.9997729214698883, + "pass@32": 0.9999890704069151, + "pass@64": 0.9999999971219998, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963711168639053, + "pass@2": 0.9970891039230303, + "pass@4": 0.9973102368306034, + "pass@8": 0.9975660626410165, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873414855072464, + "pass@2": 0.990341136026475, + "pass@4": 0.9926458491601591, + "pass@8": 0.9944704905370954, + "pass@16": 0.9957758227738857, + "pass@32": 0.9971598125688518, + "pass@64": 0.9989273079949132, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867478510028653, + "pass@2": 0.9883521958125577, + "pass@4": 0.9886221656927555, + "pass@8": 0.9887177604361171, + "pass@16": 0.9888968481375355, + "pass@32": 0.9892550143266475, + "pass@64": 0.9899713467048711, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9869942196531792, + "pass@2": 0.9909770976514497, + "pass@4": 0.993617887810989, + "pass@8": 0.9958918476913389, + "pass@16": 0.9978098101358188, + "pass@32": 0.9990707212295816, + "pass@64": 0.9998278510372128, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9852992957746479, + "pass@2": 0.9926118720195188, + "pass@4": 0.9960676550114335, + "pass@8": 0.9984719701911067, + "pass@16": 0.9997337999101152, + "pass@32": 0.9999920888176553, + "pass@64": 0.9999999988794946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658495508982036, + "pass@2": 0.9738565438021595, + "pass@4": 0.977178885479884, + "pass@8": 0.9799258010494726, + "pass@16": 0.9827626400173137, + "pass@32": 0.9852917113826465, + "pass@64": 0.9880352427034073, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 374702, + "answer_accuracy": 0.9800332691663877, + "avg_response_len": 210.10234191914964, + "resp_tokens_sum": 80329689, + "pass_at_k": { + "pass@1": 0.9800332691663877, + "pass@2": 0.9861488036082874, + "pass@4": 0.9897391150302732, + "pass@8": 0.9921392342383987, + "pass@16": 0.9939123833580354, + "pass@32": 0.9952075204583134, + "pass@64": 0.9961545829739121, + "pass@128": 0.9966521593572146 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262593283582089, + "pass@2": 0.9471449347749443, + "pass@4": 0.9614218745977648, + "pass@8": 0.972186851019286, + "pass@16": 0.9802954796338725, + "pass@32": 0.9857173184696202, + "pass@64": 0.9891864099503938, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9966071428571428, + "pass@2": 0.9971263357705287, + "pass@4": 0.9973159314907064, + "pass@8": 0.9974901553188764, + "pass@16": 0.9978149606299211, + "pass@32": 0.9983970753655793, + "pass@64": 0.9992913385826772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9972359163012786, + "pass@4": 0.9981603342767286, + "pass@8": 0.9989546945154124, + "pass@16": 0.9996537420769658, + "pass@32": 0.9999711436906581, + "pass@64": 0.999999964811974, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965049342105263, + "pass@2": 0.9970519207303035, + "pass@4": 0.9970759831007965, + "pass@8": 0.9970760233917945, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9765625, + "pass@2": 0.9826594343446041, + "pass@4": 0.9864391599947065, + "pass@8": 0.989338738990234, + "pass@16": 0.9920232898813793, + "pass@32": 0.9944213659459019, + "pass@64": 0.9963388991378888, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9905855678233438, + "pass@2": 0.9928459319655231, + "pass@4": 0.9949855764283407, + "pass@8": 0.9963629428393239, + "pass@16": 0.9968050483305237, + "pass@32": 0.996845217944392, + "pass@64": 0.9968454258674141, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9834107990506329, + "pass@2": 0.9895283065882586, + "pass@4": 0.9924962614443763, + "pass@8": 0.9935128758822745, + "pass@16": 0.993667318301558, + "pass@32": 0.9936708852185112, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9775713213213213, + "pass@2": 0.9852221808422593, + "pass@4": 0.9890983199172173, + "pass@8": 0.9915133611337327, + "pass@16": 0.9931262585622738, + "pass@32": 0.9938936237385995, + "pass@64": 0.9939935625945567, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9756079180064309, + "pass@2": 0.9861373965111273, + "pass@4": 0.9905033775922705, + "pass@8": 0.9926946650327054, + "pass@16": 0.9945845597918244, + "pass@32": 0.996690380430318, + "pass@64": 0.9988078891460136, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6068109375, + "avg_response_len": 362.87332109375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6068109375, + "pass@2": 0.6781929010826631, + "pass@4": 0.7337707151106001, + "pass@8": 0.7774104621906996, + "pass@16": 0.812397911152135, + "pass@32": 0.8407001163553003, + "pass@64": 0.8634577937668082, + "pass@128": 0.882 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9254296875, + "pass@2": 0.9455980561023625, + "pass@4": 0.9575916816647916, + "pass@8": 0.9660786754437783, + "pass@16": 0.9726976088499079, + "pass@32": 0.9780525683523252, + "pass@64": 0.9822852408522206, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8876640625, + "pass@2": 0.912449803149607, + "pass@4": 0.925959796587926, + "pass@8": 0.9355666820350127, + "pass@16": 0.9435803573597504, + "pass@32": 0.9503318362515238, + "pass@64": 0.9551076648570225, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.793328125, + "pass@2": 0.8423879183070874, + "pass@4": 0.8746621142669666, + "pass@8": 0.8993848379953863, + "pass@16": 0.9196587290575087, + "pass@32": 0.9363600136699254, + "pass@64": 0.9495616737849027, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7552421875, + "pass@2": 0.819026451771654, + "pass@4": 0.8603135664604424, + "pass@8": 0.8881597259013125, + "pass@16": 0.9086765035548701, + "pass@32": 0.9247914425923657, + "pass@64": 0.9377123715286088, + "pass@128": 0.947 + }, + "15": { + "pass@1": 0.68603125, + "pass@2": 0.7612930610236227, + "pass@4": 0.8163969390701169, + "pass@8": 0.8583675259531693, + "pass@16": 0.8893081787096128, + "pass@32": 0.9109596756292865, + "pass@64": 0.9263257608539854, + "pass@128": 0.939 + }, + "16": { + "pass@1": 0.56871875, + "pass@2": 0.6653614665354327, + "pass@4": 0.7337346590738649, + "pass@8": 0.7822931078383818, + "pass@16": 0.8196891703545264, + "pass@32": 0.8515644610904193, + "pass@64": 0.8783661141839073, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.4713515625, + "pass@2": 0.569911663385827, + "pass@4": 0.6470357365016873, + "pass@8": 0.7044032241446979, + "pass@16": 0.7485911337985129, + "pass@32": 0.7843660456692181, + "pass@64": 0.8140700557347367, + "pass@128": 0.84 + }, + "18": { + "pass@1": 0.41446875, + "pass@2": 0.5168107775590551, + "pass@4": 0.6019622500937379, + "pass@8": 0.6684197409074777, + "pass@16": 0.7204470025893406, + "pass@32": 0.762843391142011, + "pass@64": 0.7975139806671178, + "pass@128": 0.825 + }, + "19": { + "pass@1": 0.284828125, + "pass@2": 0.3740754183070868, + "pass@4": 0.45741562532808394, + "pass@8": 0.5323990298189198, + "pass@16": 0.5975583484264403, + "pass@32": 0.6515244824715801, + "pass@64": 0.6956207724619008, + "pass@128": 0.733 + }, + "20": { + "pass@1": 0.281046875, + "pass@2": 0.3750143946850392, + "pass@4": 0.4626347820584924, + "pass@8": 0.5390320718688989, + "pass@16": 0.6037720788207667, + "pass@32": 0.6562072466843096, + "pass@64": 0.6980143027436173, + "pass@128": 0.733 + } + }, + "per_op_accuracy": { + "11": 0.9254296875, + "12": 0.8876640625, + "13": 0.793328125, + "14": 0.7552421875, + "15": 0.68603125, + "16": 0.56871875, + "17": 0.4713515625, + "18": 0.41446875, + "19": 0.284828125, + "20": 0.281046875 + }, + "per_op_avg_response_len": { + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 261800, + "answer_accuracy": 0.6078194650817236, + "avg_response_len": 343.85410939821696, + "resp_tokens_sum": 148104842, + "pass_at_k": { + "pass@1": 0.6078194650817236, + "pass@2": 0.6782466786980434, + "pass@4": 0.7322968574693411, + "pass@8": 0.7744592740301475, + "pass@16": 0.8084620539797583, + "pass@32": 0.8365408446463333, + "pass@64": 0.8594061058570227, + "pass@128": 0.8772659732540862 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9234684844192634, + "pass@2": 0.9449662343021572, + "pass@4": 0.956891268637454, + "pass@8": 0.9643693469153156, + "pass@16": 0.9700565621001835, + "pass@32": 0.9749692654639327, + "pass@64": 0.9787854577528349, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8851739053672316, + "pass@2": 0.9140503358690337, + "pass@4": 0.9297601067874989, + "pass@8": 0.9404788197530561, + "pass@16": 0.94967725068733, + "pass@32": 0.9586145399733833, + "pass@64": 0.9651696834736039, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.8151449843260188, + "pass@2": 0.8586179127687409, + "pass@4": 0.8867077769275696, + "pass@8": 0.9085025252861926, + "pass@16": 0.9271233872904527, + "pass@32": 0.9449082791298123, + "pass@64": 0.9610983812381588, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7620959890965732, + "pass@2": 0.8263109561655264, + "pass@4": 0.8645924224425215, + "pass@8": 0.8889128593760814, + "pass@16": 0.9094888442681421, + "pass@32": 0.928986852397342, + "pass@64": 0.9461391387088692, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6877257947976878, + "pass@2": 0.7639100262846479, + "pass@4": 0.8192633533705684, + "pass@8": 0.8602656226549914, + "pass@16": 0.8904892628915874, + "pass@32": 0.912211514532011, + "pass@64": 0.9267263236137377, + "pass@128": 0.9364161849710982 + }, + "16": { + "pass@1": 0.5722278225806452, + "pass@2": 0.6708578073406148, + "pass@4": 0.7376889657945979, + "pass@8": 0.7811584454338113, + "pass@16": 0.8123913701188059, + "pass@32": 0.8397369710307722, + "pass@64": 0.8647941957421232, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.46899837427745666, + "pass@2": 0.5686220614673886, + "pass@4": 0.6493129596155967, + "pass@8": 0.7107867389096841, + "pass@16": 0.7572407054462839, + "pass@32": 0.7920715450323713, + "pass@64": 0.8177465613909459, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.3928348214285714, + "pass@2": 0.4923548228346458, + "pass@4": 0.57355253361187, + "pass@8": 0.6381375838952569, + "pass@16": 0.6909666831451384, + "pass@32": 0.7355241528015719, + "pass@64": 0.7725518509793133, + "pass@128": 0.8028571428571428 + }, + "19": { + "pass@1": 0.2802881006006006, + "pass@2": 0.3656185713272326, + "pass@4": 0.44623301085112094, + "pass@8": 0.5207589952897529, + "pass@16": 0.5834385538957976, + "pass@32": 0.6326537813669033, + "pass@64": 0.6755366923342895, + "pass@128": 0.7147147147147147 + }, + "20": { + "pass@1": 0.27888044294294295, + "pass@2": 0.3691022469319712, + "pass@4": 0.45260455928369264, + "pass@8": 0.5256639851382408, + "pass@16": 0.5888334762511734, + "pass@32": 0.6417662504776702, + "pass@64": 0.6828783160667011, + "pass@128": 0.7177177177177178 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 262004, + "answer_accuracy": 0.60865484686292, + "avg_response_len": 358.8022250408861, + "resp_tokens_sum": 154451441, + "pass_at_k": { + "pass@1": 0.60865484686292, + "pass@2": 0.6813515348243206, + "pass@4": 0.7380243010714206, + "pass@8": 0.7818896408204202, + "pass@16": 0.8163833261403455, + "pass@32": 0.8436691369883188, + "pass@64": 0.8652473006723143, + "pass@128": 0.8819506393101397 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9304405120481928, + "pass@2": 0.9519117315719565, + "pass@4": 0.9657082994520263, + "pass@8": 0.9757115076074933, + "pass@16": 0.9825771503102283, + "pass@32": 0.9865760138899352, + "pass@64": 0.9894810140158518, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8922413793103449, + "pass@2": 0.9128235072692715, + "pass@4": 0.9231989963832886, + "pass@8": 0.9287502632219307, + "pass@16": 0.9322745427114282, + "pass@32": 0.935291197697357, + "pass@64": 0.9379938824694157, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7765449929971989, + "pass@2": 0.8236345227067203, + "pass@4": 0.8546561067996749, + "pass@8": 0.8796959667664824, + "pass@16": 0.9017360202584387, + "pass@32": 0.9197525073028779, + "pass@64": 0.9340349790683912, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7601653874269005, + "pass@2": 0.8235343423354977, + "pass@4": 0.8647729458708009, + "pass@8": 0.8932804906568209, + "pass@16": 0.9130388497560374, + "pass@32": 0.9267935902050748, + "pass@64": 0.9365736229030256, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.6853219696969697, + "pass@2": 0.7700440676449535, + "pass@4": 0.8293955920282687, + "pass@8": 0.8725181459664744, + "pass@16": 0.9028329731845267, + "pass@32": 0.9229171920966758, + "pass@64": 0.9364439826397638, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5576923076923077, + "pass@2": 0.6493164215851224, + "pass@4": 0.7176616638197999, + "pass@8": 0.7699788625723991, + "pass@16": 0.8121033872254676, + "pass@32": 0.8467035952508719, + "pass@64": 0.8723326079810598, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4888392857142857, + "pass@2": 0.5898313322247761, + "pass@4": 0.6642076815320447, + "pass@8": 0.714858050845223, + "pass@16": 0.7517769648613764, + "pass@32": 0.781466465575029, + "pass@64": 0.8059220394919001, + "pass@128": 0.8260869565217391 + }, + "18": { + "pass@1": 0.4178768382352941, + "pass@2": 0.5259965551181103, + "pass@4": 0.6213802793952226, + "pass@8": 0.6972919448114913, + "pass@16": 0.755379792189837, + "pass@32": 0.8004193580795406, + "pass@64": 0.8365682411731703, + "pass@128": 0.8647058823529412 + }, + "19": { + "pass@1": 0.28429594494047616, + "pass@2": 0.3704518624625045, + "pass@4": 0.4515365584882246, + "pass@8": 0.5245154324308915, + "pass@16": 0.5899039910455528, + "pass@32": 0.6469910765958351, + "pass@64": 0.6931867993172561, + "pass@128": 0.7261904761904762 + }, + "20": { + "pass@1": 0.2971790793413174, + "pass@2": 0.3988401150454995, + "pass@4": 0.48962655734649935, + "pass@8": 0.5629928497263833, + "pass@16": 0.6216007899549488, + "pass@32": 0.6680350459107165, + "pass@64": 0.7074075972497136, + "pass@128": 0.7425149700598802 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 252914, + "answer_accuracy": 0.6038785528728606, + "avg_response_len": 386.6174358190709, + "resp_tokens_sum": 161921568, + "pass_at_k": { + "pass@1": 0.6038785528728606, + "pass@2": 0.6748911141178825, + "pass@4": 0.7309145786120432, + "pass@8": 0.7758417795588501, + "pass@16": 0.8123493808280585, + "pass@32": 0.8419260127219851, + "pass@64": 0.865785366533651, + "pass@128": 0.886919315403423 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9223462301587302, + "pass@2": 0.9396516841644794, + "pass@4": 0.9498219314847549, + "pass@8": 0.9578415093872525, + "pass@16": 0.9652445350017391, + "pass@32": 0.9725243842288842, + "pass@64": 0.9786231035308165, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.885894495412844, + "pass@2": 0.9103525548411951, + "pass@4": 0.9245389569996405, + "pass@8": 0.9368986112374154, + "pass@16": 0.9480092706161772, + "pass@32": 0.9560379114235142, + "pass@64": 0.9609099339438009, + "pass@128": 0.963302752293578 + }, + "13": { + "pass@1": 0.7903404706790124, + "pass@2": 0.8470718503937011, + "pass@4": 0.8848459978382324, + "pass@8": 0.9121020749798062, + "pass@16": 0.9320574360481253, + "pass@32": 0.9462426775320625, + "pass@64": 0.9553111192670196, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.7437175445103857, + "pass@2": 0.8075130260052802, + "pass@4": 0.8517123186011245, + "pass@8": 0.8822456090117424, + "pass@16": 0.9034756615081073, + "pass@32": 0.918763368316569, + "pass@64": 0.930841332255865, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6849440586419753, + "pass@2": 0.749585338291047, + "pass@4": 0.8000965229462056, + "pass@8": 0.8419278776098915, + "pass@16": 0.8742716438216978, + "pass@32": 0.897443883362062, + "pass@64": 0.9155924031250283, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.576926622418879, + "pass@2": 0.6769483253199544, + "pass@4": 0.7467605772176702, + "pass@8": 0.7960808524807924, + "pass@16": 0.8342169814205239, + "pass@32": 0.8674130918519873, + "pass@64": 0.8970240946976298, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.45684299698795183, + "pass@2": 0.551935967294374, + "pass@4": 0.6280077681366633, + "pass@8": 0.6876106027403157, + "pass@16": 0.7364869488817384, + "pass@32": 0.7791478589543936, + "pass@64": 0.8181411107743274, + "pass@128": 0.858433734939759 + }, + "18": { + "pass@1": 0.43515625, + "pass@2": 0.5343476155702312, + "pass@4": 0.6127405430168001, + "pass@8": 0.6709429848652598, + "pass@16": 0.7154178520774157, + "pass@32": 0.7524753416594111, + "pass@64": 0.7828633252434822, + "pass@128": 0.8064516129032258 + }, + "19": { + "pass@1": 0.28993580060422963, + "pass@2": 0.3862616563979352, + "pass@4": 0.47463368296863195, + "pass@8": 0.5521120818448708, + "pass@16": 0.6195334410508603, + "pass@32": 0.6751110922664687, + "pass@64": 0.7182969466585594, + "pass@128": 0.7583081570996979 + }, + "20": { + "pass@1": 0.26703265765765766, + "pass@2": 0.3570292733678559, + "pass@4": 0.4455921731630394, + "pass@8": 0.5283674264542136, + "pass@16": 0.600828430583105, + "pass@32": 0.6587849247479455, + "pass@64": 0.7037287868228262, + "pass@128": 0.7387387387387387 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7832446546052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7832446546052632, + "pass@2": 0.823485016058828, + "pass@4": 0.8542858179947747, + "pass@8": 0.8783320067346951, + "pass@16": 0.8975608206007191, + "pass@32": 0.9130890874693275, + "pass@64": 0.9255968119659093, + "pass@128": 0.9357368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9203125, + "pass@2": 0.9373841043307087, + "pass@4": 0.9486343601424823, + "pass@8": 0.9568461594757247, + "pass@16": 0.9631299032632213, + "pass@32": 0.9681102653736894, + "pass@64": 0.9720156987023217, + "pass@128": 0.975 + }, + "2": { + "pass@1": 0.9981640625, + "pass@2": 0.9987837106299212, + "pass@4": 0.9990383347394075, + "pass@8": 0.999121313250048, + "pass@16": 0.9992352362042897, + "pass@32": 0.9994389763779528, + "pass@64": 0.9997519685039371, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9952421875, + "pass@2": 0.9970708661417323, + "pass@4": 0.9983012541244844, + "pass@8": 0.9991987368058538, + "pass@16": 0.9997891651186116, + "pass@32": 0.9999863538662602, + "pass@64": 0.9999999869805416, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9923984375, + "pass@2": 0.9935083661417322, + "pass@4": 0.994365091488564, + "pass@8": 0.9951879421387454, + "pass@16": 0.995978519344931, + "pass@32": 0.9968444264374606, + "pass@64": 0.9976963412230191, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97834375, + "pass@2": 0.983570989173229, + "pass@4": 0.9870388284589425, + "pass@8": 0.9898555406065586, + "pass@16": 0.9925189720292993, + "pass@32": 0.9950251327152541, + "pass@64": 0.9973750477309096, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99003125, + "pass@2": 0.9921589566929134, + "pass@4": 0.9934118787964004, + "pass@8": 0.9940810662118578, + "pass@16": 0.9944445402431988, + "pass@32": 0.9948313926560495, + "pass@64": 0.9953779527558758, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.985546875, + "pass@2": 0.9910484744094491, + "pass@4": 0.994010423415823, + "pass@8": 0.9954839757776164, + "pass@16": 0.9962401880505216, + "pass@32": 0.9966784691049091, + "pass@64": 0.9969404364588756, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9819921875, + "pass@2": 0.9892123523622052, + "pass@4": 0.9925807832770904, + "pass@8": 0.9947584005374547, + "pass@16": 0.9960946946258159, + "pass@32": 0.9967326365666029, + "pass@64": 0.9969710370484877, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9715078125, + "pass@2": 0.9815484744094494, + "pass@4": 0.9863424363517062, + "pass@8": 0.9896703712481156, + "pass@16": 0.9922452610125915, + "pass@32": 0.9940438452658681, + "pass@64": 0.9956330202802935, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9254296875, + "pass@2": 0.9455980561023625, + "pass@4": 0.9575916816647916, + "pass@8": 0.9660786754437783, + "pass@16": 0.9726976088499079, + "pass@32": 0.9780525683523252, + "pass@64": 0.9822852408522206, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8876640625, + "pass@2": 0.912449803149607, + "pass@4": 0.925959796587926, + "pass@8": 0.9355666820350127, + "pass@16": 0.9435803573597504, + "pass@32": 0.9503318362515238, + "pass@64": 0.9551076648570225, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.793328125, + "pass@2": 0.8423879183070874, + "pass@4": 0.8746621142669666, + "pass@8": 0.8993848379953863, + "pass@16": 0.9196587290575087, + "pass@32": 0.9363600136699254, + "pass@64": 0.9495616737849027, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7552421875, + "pass@2": 0.819026451771654, + "pass@4": 0.8603135664604424, + "pass@8": 0.8881597259013125, + "pass@16": 0.9086765035548701, + "pass@32": 0.9247914425923657, + "pass@64": 0.9377123715286088, + "pass@128": 0.947 + }, + "15": { + "pass@1": 0.68603125, + "pass@2": 0.7612930610236227, + "pass@4": 0.8163969390701169, + "pass@8": 0.8583675259531693, + "pass@16": 0.8893081787096128, + "pass@32": 0.9109596756292865, + "pass@64": 0.9263257608539854, + "pass@128": 0.939 + }, + "16": { + "pass@1": 0.56871875, + "pass@2": 0.6653614665354327, + "pass@4": 0.7337346590738649, + "pass@8": 0.7822931078383818, + "pass@16": 0.8196891703545264, + "pass@32": 0.8515644610904193, + "pass@64": 0.8783661141839073, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.4713515625, + "pass@2": 0.569911663385827, + "pass@4": 0.6470357365016873, + "pass@8": 0.7044032241446979, + "pass@16": 0.7485911337985129, + "pass@32": 0.7843660456692181, + "pass@64": 0.8140700557347367, + "pass@128": 0.84 + }, + "18": { + "pass@1": 0.41446875, + "pass@2": 0.5168107775590551, + "pass@4": 0.6019622500937379, + "pass@8": 0.6684197409074777, + "pass@16": 0.7204470025893406, + "pass@32": 0.762843391142011, + "pass@64": 0.7975139806671178, + "pass@128": 0.825 + }, + "19": { + "pass@1": 0.284828125, + "pass@2": 0.3740754183070868, + "pass@4": 0.45741562532808394, + "pass@8": 0.5323990298189198, + "pass@16": 0.5975583484264403, + "pass@32": 0.6515244824715801, + "pass@64": 0.6956207724619008, + "pass@128": 0.733 + }, + "20": { + "pass@1": 0.281046875, + "pass@2": 0.3750143946850392, + "pass@4": 0.4626347820584924, + "pass@8": 0.5390320718688989, + "pass@16": 0.6037720788207667, + "pass@32": 0.6562072466843096, + "pass@64": 0.6980143027436173, + "pass@128": 0.733 + } + }, + "per_op_accuracy": { + "10": 0.9203125, + "2": 0.9981640625, + "3": 0.9952421875, + "4": 0.9923984375, + "5": 0.97834375, + "6": 0.99003125, + "7": 0.985546875, + "8": 0.9819921875, + "9": 0.9715078125, + "11": 0.9254296875, + "12": 0.8876640625, + "13": 0.793328125, + "14": 0.7552421875, + "15": 0.68603125, + "16": 0.56871875, + "17": 0.4713515625, + "18": 0.41446875, + "19": 0.284828125, + "20": 0.281046875 + }, + "per_op_avg_response_len": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375, + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375, + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 632605, + "answer_accuracy": 0.7812561749130572, + "avg_response_len": 271.58874955540625, + "resp_tokens_sum": 219913015, + "pass_at_k": { + "pass@1": 0.7812561749130572, + "pass@2": 0.8215596962977374, + "pass@4": 0.8519572684124993, + "pass@8": 0.8755206334283433, + "pass@16": 0.8944649819697402, + "pass@32": 0.9100665792514822, + "pass@64": 0.9227670820634137, + "pass@128": 0.9326588681631363 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231425147492626, + "pass@2": 0.9383285572433974, + "pass@4": 0.9473531416979957, + "pass@8": 0.9537757890092359, + "pass@16": 0.9589878410934526, + "pass@32": 0.9634827340913612, + "pass@64": 0.9671551508567272, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.999123831775701, + "pass@2": 0.999758536316138, + "pass@4": 0.9999827985870926, + "pass@8": 0.9999999340639316, + "pass@16": 0.999999999999756, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9958428899082569, + "pass@2": 0.9977124419080643, + "pass@4": 0.9987991609993705, + "pass@8": 0.9995675174167592, + "pass@16": 0.9999476070371531, + "pass@32": 0.9999995668075918, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9838134765625, + "pass@2": 0.9859390378937007, + "pass@4": 0.9883570163104611, + "pass@8": 0.9906581655190253, + "pass@16": 0.9926442820736442, + "pass@32": 0.9945717745930672, + "pass@64": 0.9963074639597295, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9704117063492064, + "pass@2": 0.9771399668791401, + "pass@4": 0.9815450669856742, + "pass@8": 0.9853588892526388, + "pass@16": 0.9894869670242535, + "pass@32": 0.9933388347853764, + "pass@64": 0.9967933357643882, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992936002994012, + "pass@2": 0.9954846704229336, + "pass@4": 0.9969230995264813, + "pass@8": 0.9975195058071472, + "pass@16": 0.9980010177318054, + "pass@32": 0.9987468819391533, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860623150887574, + "pass@2": 0.9925427625448447, + "pass@4": 0.9958278568544315, + "pass@8": 0.9969092535432367, + "pass@16": 0.9970388200006978, + "pass@32": 0.9970414196166396, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9829477163461539, + "pass@2": 0.9896030499192406, + "pass@4": 0.9923302090243528, + "pass@8": 0.9939964803271766, + "pass@16": 0.9951222806297688, + "pass@32": 0.9960540651646855, + "pass@64": 0.9967025035329475, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9732394366197183, + "pass@2": 0.9847652351114562, + "pass@4": 0.9913186990710667, + "pass@8": 0.9961890503448468, + "pass@16": 0.9991175807085956, + "pass@32": 0.9999597333809449, + "pass@64": 0.9999999878674508, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9234684844192634, + "pass@2": 0.9449662343021572, + "pass@4": 0.956891268637454, + "pass@8": 0.9643693469153156, + "pass@16": 0.9700565621001835, + "pass@32": 0.9749692654639327, + "pass@64": 0.9787854577528349, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8851739053672316, + "pass@2": 0.9140503358690337, + "pass@4": 0.9297601067874989, + "pass@8": 0.9404788197530561, + "pass@16": 0.94967725068733, + "pass@32": 0.9586145399733833, + "pass@64": 0.9651696834736039, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.8151449843260188, + "pass@2": 0.8586179127687409, + "pass@4": 0.8867077769275696, + "pass@8": 0.9085025252861926, + "pass@16": 0.9271233872904527, + "pass@32": 0.9449082791298123, + "pass@64": 0.9610983812381588, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7620959890965732, + "pass@2": 0.8263109561655264, + "pass@4": 0.8645924224425215, + "pass@8": 0.8889128593760814, + "pass@16": 0.9094888442681421, + "pass@32": 0.928986852397342, + "pass@64": 0.9461391387088692, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6877257947976878, + "pass@2": 0.7639100262846479, + "pass@4": 0.8192633533705684, + "pass@8": 0.8602656226549914, + "pass@16": 0.8904892628915874, + "pass@32": 0.912211514532011, + "pass@64": 0.9267263236137377, + "pass@128": 0.9364161849710982 + }, + "16": { + "pass@1": 0.5722278225806452, + "pass@2": 0.6708578073406148, + "pass@4": 0.7376889657945979, + "pass@8": 0.7811584454338113, + "pass@16": 0.8123913701188059, + "pass@32": 0.8397369710307722, + "pass@64": 0.8647941957421232, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.46899837427745666, + "pass@2": 0.5686220614673886, + "pass@4": 0.6493129596155967, + "pass@8": 0.7107867389096841, + "pass@16": 0.7572407054462839, + "pass@32": 0.7920715450323713, + "pass@64": 0.8177465613909459, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.3928348214285714, + "pass@2": 0.4923548228346458, + "pass@4": 0.57355253361187, + "pass@8": 0.6381375838952569, + "pass@16": 0.6909666831451384, + "pass@32": 0.7355241528015719, + "pass@64": 0.7725518509793133, + "pass@128": 0.8028571428571428 + }, + "19": { + "pass@1": 0.2802881006006006, + "pass@2": 0.3656185713272326, + "pass@4": 0.44623301085112094, + "pass@8": 0.5207589952897529, + "pass@16": 0.5834385538957976, + "pass@32": 0.6326537813669033, + "pass@64": 0.6755366923342895, + "pass@128": 0.7147147147147147 + }, + "20": { + "pass@1": 0.27888044294294295, + "pass@2": 0.3691022469319712, + "pass@4": 0.45260455928369264, + "pass@8": 0.5256639851382408, + "pass@16": 0.5888334762511734, + "pass@32": 0.6417662504776702, + "pass@64": 0.6828783160667011, + "pass@128": 0.7177177177177178 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644630, + "answer_accuracy": 0.7850618667186282, + "avg_response_len": 281.62891294816836, + "resp_tokens_sum": 231251133, + "pass_at_k": { + "pass@1": 0.7850618667186282, + "pass@2": 0.8254338993868824, + "pass@4": 0.8564373537193234, + "pass@8": 0.880388215978897, + "pass@16": 0.8992122606879791, + "pass@32": 0.9141301175895304, + "pass@64": 0.9260407494275895, + "pass@128": 0.9353078721745908 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9112586273006135, + "pass@2": 0.9263716879619345, + "pass@4": 0.9368261261245718, + "pass@8": 0.9442747604605302, + "pass@16": 0.9497976670404737, + "pass@32": 0.954829192728206, + "pass@64": 0.9594253227868381, + "pass@128": 0.9631901840490797 + }, + "2": { + "pass@1": 0.9988839285714286, + "pass@2": 0.999595753655793, + "pass@4": 0.9999491804595854, + "pass@8": 0.9999993314708793, + "pass@16": 0.9999999999510504, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.993655303030303, + "pass@2": 0.9962635707468384, + "pass@4": 0.9979543449114315, + "pass@8": 0.9990869648235982, + "pass@16": 0.9997729214698883, + "pass@32": 0.9999890704069151, + "pass@64": 0.9999999971219998, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963711168639053, + "pass@2": 0.9970891039230303, + "pass@4": 0.9973102368306034, + "pass@8": 0.9975660626410165, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873414855072464, + "pass@2": 0.990341136026475, + "pass@4": 0.9926458491601591, + "pass@8": 0.9944704905370954, + "pass@16": 0.9957758227738857, + "pass@32": 0.9971598125688518, + "pass@64": 0.9989273079949132, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867478510028653, + "pass@2": 0.9883521958125577, + "pass@4": 0.9886221656927555, + "pass@8": 0.9887177604361171, + "pass@16": 0.9888968481375355, + "pass@32": 0.9892550143266475, + "pass@64": 0.9899713467048711, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9869942196531792, + "pass@2": 0.9909770976514497, + "pass@4": 0.993617887810989, + "pass@8": 0.9958918476913389, + "pass@16": 0.9978098101358188, + "pass@32": 0.9990707212295816, + "pass@64": 0.9998278510372128, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9852992957746479, + "pass@2": 0.9926118720195188, + "pass@4": 0.9960676550114335, + "pass@8": 0.9984719701911067, + "pass@16": 0.9997337999101152, + "pass@32": 0.9999920888176553, + "pass@64": 0.9999999988794946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658495508982036, + "pass@2": 0.9738565438021595, + "pass@4": 0.977178885479884, + "pass@8": 0.9799258010494726, + "pass@16": 0.9827626400173137, + "pass@32": 0.9852917113826465, + "pass@64": 0.9880352427034073, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9304405120481928, + "pass@2": 0.9519117315719565, + "pass@4": 0.9657082994520263, + "pass@8": 0.9757115076074933, + "pass@16": 0.9825771503102283, + "pass@32": 0.9865760138899352, + "pass@64": 0.9894810140158518, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8922413793103449, + "pass@2": 0.9128235072692715, + "pass@4": 0.9231989963832886, + "pass@8": 0.9287502632219307, + "pass@16": 0.9322745427114282, + "pass@32": 0.935291197697357, + "pass@64": 0.9379938824694157, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7765449929971989, + "pass@2": 0.8236345227067203, + "pass@4": 0.8546561067996749, + "pass@8": 0.8796959667664824, + "pass@16": 0.9017360202584387, + "pass@32": 0.9197525073028779, + "pass@64": 0.9340349790683912, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7601653874269005, + "pass@2": 0.8235343423354977, + "pass@4": 0.8647729458708009, + "pass@8": 0.8932804906568209, + "pass@16": 0.9130388497560374, + "pass@32": 0.9267935902050748, + "pass@64": 0.9365736229030256, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.6853219696969697, + "pass@2": 0.7700440676449535, + "pass@4": 0.8293955920282687, + "pass@8": 0.8725181459664744, + "pass@16": 0.9028329731845267, + "pass@32": 0.9229171920966758, + "pass@64": 0.9364439826397638, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5576923076923077, + "pass@2": 0.6493164215851224, + "pass@4": 0.7176616638197999, + "pass@8": 0.7699788625723991, + "pass@16": 0.8121033872254676, + "pass@32": 0.8467035952508719, + "pass@64": 0.8723326079810598, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4888392857142857, + "pass@2": 0.5898313322247761, + "pass@4": 0.6642076815320447, + "pass@8": 0.714858050845223, + "pass@16": 0.7517769648613764, + "pass@32": 0.781466465575029, + "pass@64": 0.8059220394919001, + "pass@128": 0.8260869565217391 + }, + "18": { + "pass@1": 0.4178768382352941, + "pass@2": 0.5259965551181103, + "pass@4": 0.6213802793952226, + "pass@8": 0.6972919448114913, + "pass@16": 0.755379792189837, + "pass@32": 0.8004193580795406, + "pass@64": 0.8365682411731703, + "pass@128": 0.8647058823529412 + }, + "19": { + "pass@1": 0.28429594494047616, + "pass@2": 0.3704518624625045, + "pass@4": 0.4515365584882246, + "pass@8": 0.5245154324308915, + "pass@16": 0.5899039910455528, + "pass@32": 0.6469910765958351, + "pass@64": 0.6931867993172561, + "pass@128": 0.7261904761904762 + }, + "20": { + "pass@1": 0.2971790793413174, + "pass@2": 0.3988401150454995, + "pass@4": 0.48962655734649935, + "pass@8": 0.5629928497263833, + "pass@16": 0.6216007899549488, + "pass@32": 0.6680350459107165, + "pass@64": 0.7074075972497136, + "pass@128": 0.7425149700598802 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 627616, + "answer_accuracy": 0.7833919156414763, + "avg_response_len": 302.37864599976035, + "resp_tokens_sum": 242251257, + "pass_at_k": { + "pass@1": 0.7833919156414763, + "pass@2": 0.8234334880606494, + "pass@4": 0.8544341328988612, + "pass@8": 0.8790660161985339, + "pass@16": 0.8989971981402587, + "pass@32": 0.9150769735157885, + "pass@64": 0.9280018307463214, + "pass@128": 0.9392874261064068 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262593283582089, + "pass@2": 0.9471449347749443, + "pass@4": 0.9614218745977648, + "pass@8": 0.972186851019286, + "pass@16": 0.9802954796338725, + "pass@32": 0.9857173184696202, + "pass@64": 0.9891864099503938, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9966071428571428, + "pass@2": 0.9971263357705287, + "pass@4": 0.9973159314907064, + "pass@8": 0.9974901553188764, + "pass@16": 0.9978149606299211, + "pass@32": 0.9983970753655793, + "pass@64": 0.9992913385826772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9972359163012786, + "pass@4": 0.9981603342767286, + "pass@8": 0.9989546945154124, + "pass@16": 0.9996537420769658, + "pass@32": 0.9999711436906581, + "pass@64": 0.999999964811974, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965049342105263, + "pass@2": 0.9970519207303035, + "pass@4": 0.9970759831007965, + "pass@8": 0.9970760233917945, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9765625, + "pass@2": 0.9826594343446041, + "pass@4": 0.9864391599947065, + "pass@8": 0.989338738990234, + "pass@16": 0.9920232898813793, + "pass@32": 0.9944213659459019, + "pass@64": 0.9963388991378888, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9905855678233438, + "pass@2": 0.9928459319655231, + "pass@4": 0.9949855764283407, + "pass@8": 0.9963629428393239, + "pass@16": 0.9968050483305237, + "pass@32": 0.996845217944392, + "pass@64": 0.9968454258674141, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9834107990506329, + "pass@2": 0.9895283065882586, + "pass@4": 0.9924962614443763, + "pass@8": 0.9935128758822745, + "pass@16": 0.993667318301558, + "pass@32": 0.9936708852185112, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9775713213213213, + "pass@2": 0.9852221808422593, + "pass@4": 0.9890983199172173, + "pass@8": 0.9915133611337327, + "pass@16": 0.9931262585622738, + "pass@32": 0.9938936237385995, + "pass@64": 0.9939935625945567, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9756079180064309, + "pass@2": 0.9861373965111273, + "pass@4": 0.9905033775922705, + "pass@8": 0.9926946650327054, + "pass@16": 0.9945845597918244, + "pass@32": 0.996690380430318, + "pass@64": 0.9988078891460136, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9223462301587302, + "pass@2": 0.9396516841644794, + "pass@4": 0.9498219314847549, + "pass@8": 0.9578415093872525, + "pass@16": 0.9652445350017391, + "pass@32": 0.9725243842288842, + "pass@64": 0.9786231035308165, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.885894495412844, + "pass@2": 0.9103525548411951, + "pass@4": 0.9245389569996405, + "pass@8": 0.9368986112374154, + "pass@16": 0.9480092706161772, + "pass@32": 0.9560379114235142, + "pass@64": 0.9609099339438009, + "pass@128": 0.963302752293578 + }, + "13": { + "pass@1": 0.7903404706790124, + "pass@2": 0.8470718503937011, + "pass@4": 0.8848459978382324, + "pass@8": 0.9121020749798062, + "pass@16": 0.9320574360481253, + "pass@32": 0.9462426775320625, + "pass@64": 0.9553111192670196, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.7437175445103857, + "pass@2": 0.8075130260052802, + "pass@4": 0.8517123186011245, + "pass@8": 0.8822456090117424, + "pass@16": 0.9034756615081073, + "pass@32": 0.918763368316569, + "pass@64": 0.930841332255865, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6849440586419753, + "pass@2": 0.749585338291047, + "pass@4": 0.8000965229462056, + "pass@8": 0.8419278776098915, + "pass@16": 0.8742716438216978, + "pass@32": 0.897443883362062, + "pass@64": 0.9155924031250283, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.576926622418879, + "pass@2": 0.6769483253199544, + "pass@4": 0.7467605772176702, + "pass@8": 0.7960808524807924, + "pass@16": 0.8342169814205239, + "pass@32": 0.8674130918519873, + "pass@64": 0.8970240946976298, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.45684299698795183, + "pass@2": 0.551935967294374, + "pass@4": 0.6280077681366633, + "pass@8": 0.6876106027403157, + "pass@16": 0.7364869488817384, + "pass@32": 0.7791478589543936, + "pass@64": 0.8181411107743274, + "pass@128": 0.858433734939759 + }, + "18": { + "pass@1": 0.43515625, + "pass@2": 0.5343476155702312, + "pass@4": 0.6127405430168001, + "pass@8": 0.6709429848652598, + "pass@16": 0.7154178520774157, + "pass@32": 0.7524753416594111, + "pass@64": 0.7828633252434822, + "pass@128": 0.8064516129032258 + }, + "19": { + "pass@1": 0.28993580060422963, + "pass@2": 0.3862616563979352, + "pass@4": 0.47463368296863195, + "pass@8": 0.5521120818448708, + "pass@16": 0.6195334410508603, + "pass@32": 0.6751110922664687, + "pass@64": 0.7182969466585594, + "pass@128": 0.7583081570996979 + }, + "20": { + "pass@1": 0.26703265765765766, + "pass@2": 0.3570292733678559, + "pass@4": 0.4455921731630394, + "pass@8": 0.5283674264542136, + "pass@16": 0.600828430583105, + "pass@32": 0.6587849247479455, + "pass@64": 0.7037287868228262, + "pass@128": 0.7387387387387387 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..592f7e41e5bb2c06a65a413e5fd6277473186c6f --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e25f688620add169d2ed6b61ccc8f0d5ee96a971c79942851d6436e1053d87b +size 2684826794 diff --git a/results/composition/SDAR-100M-block8-epoch1-op11-14/summary.csv b/results/composition/SDAR-100M-block8-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..3721bdf8918b392ffd9206d85fc7e20a288f7a44 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op11-14,0.9792821180555555,0.6068109375,0.7832446546052632,0.0,0.0,0.0,198.730515625,362.87332109375,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch1-op11-14/summary.json b/results/composition/SDAR-100M-block8-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a9f646ea9ab8162e9027924576d3c00141881e4c --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op11-14", + "id_acc": 0.9792821180555555, + "ood_acc": 0.6068109375, + "total_acc": 0.7832446546052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.730515625, + "ood_avg_resp_len": 362.87332109375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 632605, + "answer_accuracy": 0.7812561749130572, + "avg_response_len": 271.58874955540625, + "resp_tokens_sum": 219913015, + "pass_at_k": { + "pass@1": 0.7812561749130572, + "pass@2": 0.8215596962977374, + "pass@4": 0.8519572684124993, + "pass@8": 0.8755206334283433, + "pass@16": 0.8944649819697402, + "pass@32": 0.9100665792514822, + "pass@64": 0.9227670820634137, + "pass@128": 0.9326588681631363 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231425147492626, + "pass@2": 0.9383285572433974, + "pass@4": 0.9473531416979957, + "pass@8": 0.9537757890092359, + "pass@16": 0.9589878410934526, + "pass@32": 0.9634827340913612, + "pass@64": 0.9671551508567272, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.999123831775701, + "pass@2": 0.999758536316138, + "pass@4": 0.9999827985870926, + "pass@8": 0.9999999340639316, + "pass@16": 0.999999999999756, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9958428899082569, + "pass@2": 0.9977124419080643, + "pass@4": 0.9987991609993705, + "pass@8": 0.9995675174167592, + "pass@16": 0.9999476070371531, + "pass@32": 0.9999995668075918, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9838134765625, + "pass@2": 0.9859390378937007, + "pass@4": 0.9883570163104611, + "pass@8": 0.9906581655190253, + "pass@16": 0.9926442820736442, + "pass@32": 0.9945717745930672, + "pass@64": 0.9963074639597295, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9704117063492064, + "pass@2": 0.9771399668791401, + "pass@4": 0.9815450669856742, + "pass@8": 0.9853588892526388, + "pass@16": 0.9894869670242535, + "pass@32": 0.9933388347853764, + "pass@64": 0.9967933357643882, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992936002994012, + "pass@2": 0.9954846704229336, + "pass@4": 0.9969230995264813, + "pass@8": 0.9975195058071472, + "pass@16": 0.9980010177318054, + "pass@32": 0.9987468819391533, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860623150887574, + "pass@2": 0.9925427625448447, + "pass@4": 0.9958278568544315, + "pass@8": 0.9969092535432367, + "pass@16": 0.9970388200006978, + "pass@32": 0.9970414196166396, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9829477163461539, + "pass@2": 0.9896030499192406, + "pass@4": 0.9923302090243528, + "pass@8": 0.9939964803271766, + "pass@16": 0.9951222806297688, + "pass@32": 0.9960540651646855, + "pass@64": 0.9967025035329475, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9732394366197183, + "pass@2": 0.9847652351114562, + "pass@4": 0.9913186990710667, + "pass@8": 0.9961890503448468, + "pass@16": 0.9991175807085956, + "pass@32": 0.9999597333809449, + "pass@64": 0.9999999878674508, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9234684844192634, + "pass@2": 0.9449662343021572, + "pass@4": 0.956891268637454, + "pass@8": 0.9643693469153156, + "pass@16": 0.9700565621001835, + "pass@32": 0.9749692654639327, + "pass@64": 0.9787854577528349, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8851739053672316, + "pass@2": 0.9140503358690337, + "pass@4": 0.9297601067874989, + "pass@8": 0.9404788197530561, + "pass@16": 0.94967725068733, + "pass@32": 0.9586145399733833, + "pass@64": 0.9651696834736039, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.8151449843260188, + "pass@2": 0.8586179127687409, + "pass@4": 0.8867077769275696, + "pass@8": 0.9085025252861926, + "pass@16": 0.9271233872904527, + "pass@32": 0.9449082791298123, + "pass@64": 0.9610983812381588, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7620959890965732, + "pass@2": 0.8263109561655264, + "pass@4": 0.8645924224425215, + "pass@8": 0.8889128593760814, + "pass@16": 0.9094888442681421, + "pass@32": 0.928986852397342, + "pass@64": 0.9461391387088692, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6877257947976878, + "pass@2": 0.7639100262846479, + "pass@4": 0.8192633533705684, + "pass@8": 0.8602656226549914, + "pass@16": 0.8904892628915874, + "pass@32": 0.912211514532011, + "pass@64": 0.9267263236137377, + "pass@128": 0.9364161849710982 + }, + "16": { + "pass@1": 0.5722278225806452, + "pass@2": 0.6708578073406148, + "pass@4": 0.7376889657945979, + "pass@8": 0.7811584454338113, + "pass@16": 0.8123913701188059, + "pass@32": 0.8397369710307722, + "pass@64": 0.8647941957421232, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.46899837427745666, + "pass@2": 0.5686220614673886, + "pass@4": 0.6493129596155967, + "pass@8": 0.7107867389096841, + "pass@16": 0.7572407054462839, + "pass@32": 0.7920715450323713, + "pass@64": 0.8177465613909459, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.3928348214285714, + "pass@2": 0.4923548228346458, + "pass@4": 0.57355253361187, + "pass@8": 0.6381375838952569, + "pass@16": 0.6909666831451384, + "pass@32": 0.7355241528015719, + "pass@64": 0.7725518509793133, + "pass@128": 0.8028571428571428 + }, + "19": { + "pass@1": 0.2802881006006006, + "pass@2": 0.3656185713272326, + "pass@4": 0.44623301085112094, + "pass@8": 0.5207589952897529, + "pass@16": 0.5834385538957976, + "pass@32": 0.6326537813669033, + "pass@64": 0.6755366923342895, + "pass@128": 0.7147147147147147 + }, + "20": { + "pass@1": 0.27888044294294295, + "pass@2": 0.3691022469319712, + "pass@4": 0.45260455928369264, + "pass@8": 0.5256639851382408, + "pass@16": 0.5888334762511734, + "pass@32": 0.6417662504776702, + "pass@64": 0.6828783160667011, + "pass@128": 0.7177177177177178 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644630, + "answer_accuracy": 0.7850618667186282, + "avg_response_len": 281.62891294816836, + "resp_tokens_sum": 231251133, + "pass_at_k": { + "pass@1": 0.7850618667186282, + "pass@2": 0.8254338993868824, + "pass@4": 0.8564373537193234, + "pass@8": 0.880388215978897, + "pass@16": 0.8992122606879791, + "pass@32": 0.9141301175895304, + "pass@64": 0.9260407494275895, + "pass@128": 0.9353078721745908 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9112586273006135, + "pass@2": 0.9263716879619345, + "pass@4": 0.9368261261245718, + "pass@8": 0.9442747604605302, + "pass@16": 0.9497976670404737, + "pass@32": 0.954829192728206, + "pass@64": 0.9594253227868381, + "pass@128": 0.9631901840490797 + }, + "2": { + "pass@1": 0.9988839285714286, + "pass@2": 0.999595753655793, + "pass@4": 0.9999491804595854, + "pass@8": 0.9999993314708793, + "pass@16": 0.9999999999510504, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.993655303030303, + "pass@2": 0.9962635707468384, + "pass@4": 0.9979543449114315, + "pass@8": 0.9990869648235982, + "pass@16": 0.9997729214698883, + "pass@32": 0.9999890704069151, + "pass@64": 0.9999999971219998, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963711168639053, + "pass@2": 0.9970891039230303, + "pass@4": 0.9973102368306034, + "pass@8": 0.9975660626410165, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873414855072464, + "pass@2": 0.990341136026475, + "pass@4": 0.9926458491601591, + "pass@8": 0.9944704905370954, + "pass@16": 0.9957758227738857, + "pass@32": 0.9971598125688518, + "pass@64": 0.9989273079949132, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867478510028653, + "pass@2": 0.9883521958125577, + "pass@4": 0.9886221656927555, + "pass@8": 0.9887177604361171, + "pass@16": 0.9888968481375355, + "pass@32": 0.9892550143266475, + "pass@64": 0.9899713467048711, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9869942196531792, + "pass@2": 0.9909770976514497, + "pass@4": 0.993617887810989, + "pass@8": 0.9958918476913389, + "pass@16": 0.9978098101358188, + "pass@32": 0.9990707212295816, + "pass@64": 0.9998278510372128, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9852992957746479, + "pass@2": 0.9926118720195188, + "pass@4": 0.9960676550114335, + "pass@8": 0.9984719701911067, + "pass@16": 0.9997337999101152, + "pass@32": 0.9999920888176553, + "pass@64": 0.9999999988794946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658495508982036, + "pass@2": 0.9738565438021595, + "pass@4": 0.977178885479884, + "pass@8": 0.9799258010494726, + "pass@16": 0.9827626400173137, + "pass@32": 0.9852917113826465, + "pass@64": 0.9880352427034073, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9304405120481928, + "pass@2": 0.9519117315719565, + "pass@4": 0.9657082994520263, + "pass@8": 0.9757115076074933, + "pass@16": 0.9825771503102283, + "pass@32": 0.9865760138899352, + "pass@64": 0.9894810140158518, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8922413793103449, + "pass@2": 0.9128235072692715, + "pass@4": 0.9231989963832886, + "pass@8": 0.9287502632219307, + "pass@16": 0.9322745427114282, + "pass@32": 0.935291197697357, + "pass@64": 0.9379938824694157, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7765449929971989, + "pass@2": 0.8236345227067203, + "pass@4": 0.8546561067996749, + "pass@8": 0.8796959667664824, + "pass@16": 0.9017360202584387, + "pass@32": 0.9197525073028779, + "pass@64": 0.9340349790683912, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7601653874269005, + "pass@2": 0.8235343423354977, + "pass@4": 0.8647729458708009, + "pass@8": 0.8932804906568209, + "pass@16": 0.9130388497560374, + "pass@32": 0.9267935902050748, + "pass@64": 0.9365736229030256, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.6853219696969697, + "pass@2": 0.7700440676449535, + "pass@4": 0.8293955920282687, + "pass@8": 0.8725181459664744, + "pass@16": 0.9028329731845267, + "pass@32": 0.9229171920966758, + "pass@64": 0.9364439826397638, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5576923076923077, + "pass@2": 0.6493164215851224, + "pass@4": 0.7176616638197999, + "pass@8": 0.7699788625723991, + "pass@16": 0.8121033872254676, + "pass@32": 0.8467035952508719, + "pass@64": 0.8723326079810598, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4888392857142857, + "pass@2": 0.5898313322247761, + "pass@4": 0.6642076815320447, + "pass@8": 0.714858050845223, + "pass@16": 0.7517769648613764, + "pass@32": 0.781466465575029, + "pass@64": 0.8059220394919001, + "pass@128": 0.8260869565217391 + }, + "18": { + "pass@1": 0.4178768382352941, + "pass@2": 0.5259965551181103, + "pass@4": 0.6213802793952226, + "pass@8": 0.6972919448114913, + "pass@16": 0.755379792189837, + "pass@32": 0.8004193580795406, + "pass@64": 0.8365682411731703, + "pass@128": 0.8647058823529412 + }, + "19": { + "pass@1": 0.28429594494047616, + "pass@2": 0.3704518624625045, + "pass@4": 0.4515365584882246, + "pass@8": 0.5245154324308915, + "pass@16": 0.5899039910455528, + "pass@32": 0.6469910765958351, + "pass@64": 0.6931867993172561, + "pass@128": 0.7261904761904762 + }, + "20": { + "pass@1": 0.2971790793413174, + "pass@2": 0.3988401150454995, + "pass@4": 0.48962655734649935, + "pass@8": 0.5629928497263833, + "pass@16": 0.6216007899549488, + "pass@32": 0.6680350459107165, + "pass@64": 0.7074075972497136, + "pass@128": 0.7425149700598802 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 627616, + "answer_accuracy": 0.7833919156414763, + "avg_response_len": 302.37864599976035, + "resp_tokens_sum": 242251257, + "pass_at_k": { + "pass@1": 0.7833919156414763, + "pass@2": 0.8234334880606494, + "pass@4": 0.8544341328988612, + "pass@8": 0.8790660161985339, + "pass@16": 0.8989971981402587, + "pass@32": 0.9150769735157885, + "pass@64": 0.9280018307463214, + "pass@128": 0.9392874261064068 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262593283582089, + "pass@2": 0.9471449347749443, + "pass@4": 0.9614218745977648, + "pass@8": 0.972186851019286, + "pass@16": 0.9802954796338725, + "pass@32": 0.9857173184696202, + "pass@64": 0.9891864099503938, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9966071428571428, + "pass@2": 0.9971263357705287, + "pass@4": 0.9973159314907064, + "pass@8": 0.9974901553188764, + "pass@16": 0.9978149606299211, + "pass@32": 0.9983970753655793, + "pass@64": 0.9992913385826772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9972359163012786, + "pass@4": 0.9981603342767286, + "pass@8": 0.9989546945154124, + "pass@16": 0.9996537420769658, + "pass@32": 0.9999711436906581, + "pass@64": 0.999999964811974, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965049342105263, + "pass@2": 0.9970519207303035, + "pass@4": 0.9970759831007965, + "pass@8": 0.9970760233917945, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9765625, + "pass@2": 0.9826594343446041, + "pass@4": 0.9864391599947065, + "pass@8": 0.989338738990234, + "pass@16": 0.9920232898813793, + "pass@32": 0.9944213659459019, + "pass@64": 0.9963388991378888, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9905855678233438, + "pass@2": 0.9928459319655231, + "pass@4": 0.9949855764283407, + "pass@8": 0.9963629428393239, + "pass@16": 0.9968050483305237, + "pass@32": 0.996845217944392, + "pass@64": 0.9968454258674141, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9834107990506329, + "pass@2": 0.9895283065882586, + "pass@4": 0.9924962614443763, + "pass@8": 0.9935128758822745, + "pass@16": 0.993667318301558, + "pass@32": 0.9936708852185112, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9775713213213213, + "pass@2": 0.9852221808422593, + "pass@4": 0.9890983199172173, + "pass@8": 0.9915133611337327, + "pass@16": 0.9931262585622738, + "pass@32": 0.9938936237385995, + "pass@64": 0.9939935625945567, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9756079180064309, + "pass@2": 0.9861373965111273, + "pass@4": 0.9905033775922705, + "pass@8": 0.9926946650327054, + "pass@16": 0.9945845597918244, + "pass@32": 0.996690380430318, + "pass@64": 0.9988078891460136, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9223462301587302, + "pass@2": 0.9396516841644794, + "pass@4": 0.9498219314847549, + "pass@8": 0.9578415093872525, + "pass@16": 0.9652445350017391, + "pass@32": 0.9725243842288842, + "pass@64": 0.9786231035308165, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.885894495412844, + "pass@2": 0.9103525548411951, + "pass@4": 0.9245389569996405, + "pass@8": 0.9368986112374154, + "pass@16": 0.9480092706161772, + "pass@32": 0.9560379114235142, + "pass@64": 0.9609099339438009, + "pass@128": 0.963302752293578 + }, + "13": { + "pass@1": 0.7903404706790124, + "pass@2": 0.8470718503937011, + "pass@4": 0.8848459978382324, + "pass@8": 0.9121020749798062, + "pass@16": 0.9320574360481253, + "pass@32": 0.9462426775320625, + "pass@64": 0.9553111192670196, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.7437175445103857, + "pass@2": 0.8075130260052802, + "pass@4": 0.8517123186011245, + "pass@8": 0.8822456090117424, + "pass@16": 0.9034756615081073, + "pass@32": 0.918763368316569, + "pass@64": 0.930841332255865, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6849440586419753, + "pass@2": 0.749585338291047, + "pass@4": 0.8000965229462056, + "pass@8": 0.8419278776098915, + "pass@16": 0.8742716438216978, + "pass@32": 0.897443883362062, + "pass@64": 0.9155924031250283, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.576926622418879, + "pass@2": 0.6769483253199544, + "pass@4": 0.7467605772176702, + "pass@8": 0.7960808524807924, + "pass@16": 0.8342169814205239, + "pass@32": 0.8674130918519873, + "pass@64": 0.8970240946976298, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.45684299698795183, + "pass@2": 0.551935967294374, + "pass@4": 0.6280077681366633, + "pass@8": 0.6876106027403157, + "pass@16": 0.7364869488817384, + "pass@32": 0.7791478589543936, + "pass@64": 0.8181411107743274, + "pass@128": 0.858433734939759 + }, + "18": { + "pass@1": 0.43515625, + "pass@2": 0.5343476155702312, + "pass@4": 0.6127405430168001, + "pass@8": 0.6709429848652598, + "pass@16": 0.7154178520774157, + "pass@32": 0.7524753416594111, + "pass@64": 0.7828633252434822, + "pass@128": 0.8064516129032258 + }, + "19": { + "pass@1": 0.28993580060422963, + "pass@2": 0.3862616563979352, + "pass@4": 0.47463368296863195, + "pass@8": 0.5521120818448708, + "pass@16": 0.6195334410508603, + "pass@32": 0.6751110922664687, + "pass@64": 0.7182969466585594, + "pass@128": 0.7583081570996979 + }, + "20": { + "pass@1": 0.26703265765765766, + "pass@2": 0.3570292733678559, + "pass@4": 0.4455921731630394, + "pass@8": 0.5283674264542136, + "pass@16": 0.600828430583105, + "pass@32": 0.6587849247479455, + "pass@64": 0.7037287868228262, + "pass@128": 0.7387387387387387 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0518c2424af58ddbe9bc61375ec010bbc04dd267 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21ed6a21177b88c71bee235f44cfb747e10a8603e0a8fb61ed3499a8cfc9f993 +size 1793385092 diff --git a/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json b/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..222f4004f6d8a0c30dfecd72fea702dd7febe5c1 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9763394097222222, + "avg_response_len": 199.7977092013889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9763394097222222, + "pass@2": 0.985418990594919, + "pass@4": 0.9903296812690076, + "pass@8": 0.9934610141867211, + "pass@16": 0.995578293766543, + "pass@32": 0.9971099645277666, + "pass@64": 0.9982153127246509, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.91640625, + "pass@2": 0.9476706446850395, + "pass@4": 0.9669583255530562, + "pass@8": 0.9788451880131132, + "pass@16": 0.9862062777304876, + "pass@32": 0.9912431276236333, + "pass@64": 0.9947722680345038, + "pass@128": 0.997 + }, + "2": { + "pass@1": 0.99846875, + "pass@2": 0.9988816437007875, + "pass@4": 0.9989923884514436, + "pass@8": 0.9989999655709787, + "pass@16": 0.9989999999997617, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.9932734375, + "pass@2": 0.9957941683070867, + "pass@4": 0.9978264966254219, + "pass@8": 0.9990544210606774, + "pass@16": 0.9996775186156238, + "pass@32": 0.9999476963247053, + "pass@64": 0.9999993318674025, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992609375, + "pass@2": 0.9941601870078742, + "pass@4": 0.995101639295088, + "pass@8": 0.9959862454184044, + "pass@16": 0.9967492275545933, + "pass@32": 0.9973881759502105, + "pass@64": 0.9979968420843298, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9737109375, + "pass@2": 0.9815353100393699, + "pass@4": 0.9869551624484438, + "pass@8": 0.9911598043008092, + "pass@16": 0.9944993273291891, + "pass@32": 0.9971205319942156, + "pass@64": 0.9989734239317924, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9908984375, + "pass@2": 0.9944271653543306, + "pass@4": 0.9961795943007125, + "pass@8": 0.9970593753784408, + "pass@16": 0.9976541287314281, + "pass@32": 0.9981434924936788, + "pass@64": 0.9984968766172793, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.983296875, + "pass@2": 0.9901711368110238, + "pass@4": 0.9935972257217848, + "pass@8": 0.9958620196079306, + "pass@16": 0.997439017820933, + "pass@32": 0.9986513545851102, + "pass@64": 0.9995945257898196, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.974046875, + "pass@2": 0.9852920767716542, + "pass@4": 0.9903052625609297, + "pass@8": 0.9934548398646773, + "pass@16": 0.9956272175270217, + "pass@32": 0.9970991849627101, + "pass@64": 0.997866112633597, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.96434375, + "pass@2": 0.9808385826771662, + "pass@4": 0.9870510364641919, + "pass@8": 0.9907272684654614, + "pass@16": 0.993351928589844, + "pass@32": 0.9953961168156369, + "pass@64": 0.9972384335631338, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.91640625, + "2": 0.99846875, + "3": 0.9932734375, + "4": 0.992609375, + "5": 0.9737109375, + "6": 0.9908984375, + "7": 0.983296875, + "8": 0.974046875, + "9": 0.96434375 + }, + "per_op_avg_response_len": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 369255, + "answer_accuracy": 0.974267033941236, + "avg_response_len": 190.63018986406618, + "resp_tokens_sum": 72250367, + "pass_at_k": { + "pass@1": 0.974267033941236, + "pass@2": 0.9841380366018093, + "pass@4": 0.9894483518207642, + "pass@8": 0.992895127635665, + "pass@16": 0.995180950765034, + "pass@32": 0.9967866199320052, + "pass@64": 0.9979591953268966, + "pass@128": 0.9989868287740629 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9147999631268436, + "pass@2": 0.9457115357814785, + "pass@4": 0.9644939742045515, + "pass@8": 0.976467977625549, + "pass@16": 0.9845557191374704, + "pass@32": 0.9904786548196389, + "pass@64": 0.9943848945020379, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9949589067278287, + "pass@2": 0.9971454435454742, + "pass@4": 0.998757460363326, + "pass@8": 0.9996935826438439, + "pass@16": 0.9999781984079241, + "pass@32": 0.9999999385362756, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985107421875, + "pass@2": 0.9878137303149606, + "pass@4": 0.9902266852385638, + "pass@8": 0.992418635965382, + "pass@16": 0.9940056498208453, + "pass@32": 0.9952528850596998, + "pass@64": 0.9968748060893958, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9661954365079365, + "pass@2": 0.9754136201724785, + "pass@4": 0.9818137741710856, + "pass@8": 0.9867755777021514, + "pass@16": 0.9913209124055069, + "pass@32": 0.9954008763900372, + "pass@64": 0.9983700744266549, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9920471556886228, + "pass@2": 0.9955804434438211, + "pass@4": 0.9972394134677776, + "pass@8": 0.998204156494888, + "pass@16": 0.999004031075878, + "pass@32": 0.9997223135643741, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9807923446745562, + "pass@2": 0.9899740250663933, + "pass@4": 0.9942786351040437, + "pass@8": 0.9966196608845793, + "pass@16": 0.9978572891506724, + "pass@32": 0.9987563633971823, + "pass@64": 0.9996389129939015, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9767878605769231, + "pass@2": 0.9883080645568344, + "pass@4": 0.9921772134252452, + "pass@8": 0.9944861816978635, + "pass@16": 0.9959239960690798, + "pass@32": 0.9966348318465099, + "pass@64": 0.9967927316152657, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9637764084507042, + "pass@2": 0.9817663857158702, + "pass@4": 0.9897242576192058, + "pass@8": 0.994632208919704, + "pass@16": 0.9970542065127949, + "pass@32": 0.9978697672017515, + "pass@64": 0.9985915429355597, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382113, + "answer_accuracy": 0.9781316554718218, + "avg_response_len": 197.60126044397117, + "resp_tokens_sum": 77194118, + "pass_at_k": { + "pass@1": 0.9781316554718218, + "pass@2": 0.9867432353639298, + "pass@4": 0.9912419731220529, + "pass@8": 0.9938570392389242, + "pass@16": 0.9955842173182657, + "pass@32": 0.9969163336213072, + "pass@64": 0.9980768353031001, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.915763995398773, + "pass@2": 0.9480459881165166, + "pass@4": 0.9681581655245539, + "pass@8": 0.9794301390879024, + "pass@16": 0.9853206794298328, + "pass@32": 0.989459399841, + "pass@64": 0.9934911833242681, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9990976443768997, + "pass@2": 0.9997371089916952, + "pass@4": 0.9999789687193356, + "pass@8": 0.9999998960323445, + "pass@16": 0.9999999999992757, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990814393939394, + "pass@2": 0.9942462270341206, + "pass@4": 0.9969956352046904, + "pass@8": 0.9983947582818798, + "pass@16": 0.9992437871465215, + "pass@32": 0.9998493181618995, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959781804733728, + "pass@2": 0.9972641872058892, + "pass@4": 0.9977194152506086, + "pass@8": 0.9982611890819908, + "pass@16": 0.9990160343548016, + "pass@32": 0.9997255997961729, + "pass@64": 0.9999908406382343, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9807065217391304, + "pass@2": 0.9866305774278213, + "pass@4": 0.9910699586464737, + "pass@8": 0.9945806050238211, + "pass@16": 0.997016986481387, + "pass@32": 0.9983197945450435, + "pass@64": 0.9992809854172511, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867030802292264, + "pass@2": 0.9902159995938904, + "pass@4": 0.9921985556783911, + "pass@8": 0.9933309007238308, + "pass@16": 0.9942316250648926, + "pass@32": 0.9949462457401639, + "pass@64": 0.9957019268812495, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9899972904624278, + "pass@2": 0.9941499590369122, + "pass@4": 0.9955985682425533, + "pass@8": 0.9965680962120179, + "pass@16": 0.9975371405733434, + "pass@32": 0.9983638978272487, + "pass@64": 0.9992831394824679, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9800836267605634, + "pass@2": 0.9910738050349339, + "pass@4": 0.9956219548260692, + "pass@8": 0.997544552113714, + "pass@16": 0.9983496167020315, + "pass@32": 0.9991228206283554, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9617795658682635, + "pass@2": 0.9778661181573863, + "pass@4": 0.9829365481635154, + "pass@8": 0.98602176677128, + "pass@16": 0.9891346058079603, + "pass@32": 0.9921804769068225, + "pass@64": 0.9949455674393275, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 373375, + "answer_accuracy": 0.9765625, + "avg_response_len": 211.12967651489788, + "resp_tokens_sum": 80722476, + "pass_at_k": { + "pass@1": 0.9765625, + "pass@2": 0.985335733058477, + "pass@4": 0.990271195082453, + "pass@8": 0.9936173321038113, + "pass@16": 0.9959661256874036, + "pass@32": 0.9976283391091408, + "pass@64": 0.9986106916015589, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9186567164179105, + "pass@2": 0.9492878863556237, + "pass@4": 0.9682844965274856, + "pass@8": 0.9806815470310305, + "pass@16": 0.988738350593314, + "pass@32": 0.9937525351988347, + "pass@64": 0.9964109344316464, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9993303571428571, + "pass@2": 0.9999089566929135, + "pass@4": 0.9999980221222345, + "pass@8": 0.9999999993609641, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940324344023324, + "pass@2": 0.9959951992608066, + "pass@4": 0.9977383308136045, + "pass@8": 0.9990797355776098, + "pass@16": 0.9998081573699138, + "pass@32": 0.999992540612001, + "pass@64": 0.9999999988420875, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962993421052632, + "pass@2": 0.9970306959985263, + "pass@4": 0.9970758411229297, + "pass@8": 0.9970760233911381, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9735753676470589, + "pass@2": 0.9820366778601207, + "pass@4": 0.9875431995632898, + "pass@8": 0.9917505546806278, + "pass@16": 0.994889351721694, + "pass@32": 0.9974968435685705, + "pass@64": 0.9992203250542475, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9943069794952681, + "pass@2": 0.9978483072108101, + "pass@4": 0.9994458494344357, + "pass@8": 0.9999580528597197, + "pass@16": 0.9999998271433355, + "pass@32": 0.999999999999561, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9786392405063291, + "pass@2": 0.9860254130120601, + "pass@4": 0.9906770330528303, + "pass@8": 0.9942785219607105, + "pass@16": 0.9968841881317372, + "pass@32": 0.9988537819893496, + "pass@64": 0.9998879966359081, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9650431681681682, + "pass@2": 0.9763025821096688, + "pass@4": 0.982883417444441, + "pass@8": 0.9881286341578839, + "pass@16": 0.9924468973701125, + "pass@32": 0.9953769252358343, + "pass@64": 0.996775794068325, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9677451768488746, + "pass@2": 0.9829718079854168, + "pass@4": 0.9884183856037287, + "pass@8": 0.9913233575477782, + "pass@16": 0.9936550705400102, + "pass@32": 0.9960259491065468, + "pass@64": 0.998156322496061, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.608040625, + "avg_response_len": 372.15247421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.608040625, + "pass@2": 0.6918071481299107, + "pass@4": 0.7545935140138644, + "pass@8": 0.8021140311865859, + "pass@16": 0.838465956835818, + "pass@32": 0.8668699739628293, + "pass@64": 0.8899108409221937, + "pass@128": 0.9093 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.89928125, + "pass@2": 0.9340199311023635, + "pass@4": 0.9530581857892761, + "pass@8": 0.9646084167445874, + "pass@16": 0.9721545862699154, + "pass@32": 0.978247602678375, + "pass@64": 0.9833447019063941, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.84053125, + "pass@2": 0.8881071604330706, + "pass@4": 0.9157314617547808, + "pass@8": 0.9342534388562534, + "pass@16": 0.9479341431792865, + "pass@32": 0.9595398724244704, + "pass@64": 0.9696723257993726, + "pass@128": 0.978 + }, + "13": { + "pass@1": 0.7338671875, + "pass@2": 0.8052672244094486, + "pass@4": 0.8534425268091481, + "pass@8": 0.8896280507676896, + "pass@16": 0.9178852960128545, + "pass@32": 0.9396084313296421, + "pass@64": 0.9551960359821134, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.7240703125, + "pass@2": 0.8020214074803148, + "pass@4": 0.8532807261904767, + "pass@8": 0.8874175863692658, + "pass@16": 0.9118375285344886, + "pass@32": 0.9301418992032527, + "pass@64": 0.9433938844205266, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.6850625, + "pass@2": 0.7675178395669306, + "pass@4": 0.8241469076677913, + "pass@8": 0.8637729210547304, + "pass@16": 0.8922964442166276, + "pass@32": 0.9133260490070577, + "pass@64": 0.9290881232929966, + "pass@128": 0.942 + }, + "16": { + "pass@1": 0.5612265625, + "pass@2": 0.667350024606298, + "pass@4": 0.743922721503563, + "pass@8": 0.799555250281722, + "pass@16": 0.8414756351517275, + "pass@32": 0.8738830725990342, + "pass@64": 0.9003686093194375, + "pass@128": 0.923 + }, + "17": { + "pass@1": 0.501125, + "pass@2": 0.6063334153543315, + "pass@4": 0.6872567897450311, + "pass@8": 0.7494785918750635, + "pass@16": 0.7974884869567112, + "pass@32": 0.8351106551153658, + "pass@64": 0.8664328729653366, + "pass@128": 0.893 + }, + "18": { + "pass@1": 0.466578125, + "pass@2": 0.571246186023622, + "pass@4": 0.6533641156730406, + "pass@8": 0.7167335427334711, + "pass@16": 0.7651675894122477, + "pass@32": 0.8020617703237342, + "pass@64": 0.831716328608351, + "pass@128": 0.858 + }, + "19": { + "pass@1": 0.3256015625, + "pass@2": 0.42588127460629893, + "pass@4": 0.5165330664604426, + "pass@8": 0.5937313861165137, + "pass@16": 0.6572152568113311, + "pass@32": 0.7092619621608254, + "pass@64": 0.7536724854580563, + "pass@128": 0.793 + }, + "20": { + "pass@1": 0.3430625, + "pass@2": 0.450327017716536, + "pass@4": 0.5451986385451821, + "pass@8": 0.6219611270666211, + "pass@16": 0.6812046018128655, + "pass@32": 0.7275184247865151, + "pass@64": 0.7662230414692883, + "pass@128": 0.799 + } + }, + "per_op_accuracy": { + "11": 0.89928125, + "12": 0.84053125, + "13": 0.7338671875, + "14": 0.7240703125, + "15": 0.6850625, + "16": 0.5612265625, + "17": 0.501125, + "18": 0.466578125, + "19": 0.3256015625, + "20": 0.3430625 + }, + "per_op_avg_response_len": { + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 262611, + "answer_accuracy": 0.6097023588410104, + "avg_response_len": 353.31013187221396, + "resp_tokens_sum": 152177740, + "pass_at_k": { + "pass@1": 0.6097023588410104, + "pass@2": 0.6933687668916978, + "pass@4": 0.7545651334816439, + "pass@8": 0.8001828025446808, + "pass@16": 0.8352171976360004, + "pass@32": 0.8633249010412943, + "pass@64": 0.8870723629289942, + "pass@128": 0.9078751857355126 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8884782223796034, + "pass@2": 0.9233639250741679, + "pass@4": 0.9423458353116624, + "pass@8": 0.9535150564785112, + "pass@16": 0.9606443331526207, + "pass@32": 0.9671079842769559, + "pass@64": 0.9738184857703525, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.831302966101695, + "pass@2": 0.8819387678499936, + "pass@4": 0.9108717917145948, + "pass@8": 0.9301873731209482, + "pass@16": 0.9445633356962888, + "pass@32": 0.9571175271859526, + "pass@64": 0.9693614586588243, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7450528996865203, + "pass@2": 0.817905672253351, + "pass@4": 0.8649554258303918, + "pass@8": 0.900549165326192, + "pass@16": 0.9298497520568195, + "pass@32": 0.9535119750141245, + "pass@64": 0.9707018746756205, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.7362733644859814, + "pass@2": 0.8120324036598229, + "pass@4": 0.8572509446833161, + "pass@8": 0.8877305547576959, + "pass@16": 0.9128675754806822, + "pass@32": 0.9344930311271058, + "pass@64": 0.9501011437979755, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.6830066835260116, + "pass@2": 0.7652260365936912, + "pass@4": 0.8216380976221894, + "pass@8": 0.8606056314618283, + "pass@16": 0.8899187889510047, + "pass@32": 0.9132096707385248, + "pass@64": 0.9310756807309161, + "pass@128": 0.9479768786127167 + }, + "16": { + "pass@1": 0.5638104838709678, + "pass@2": 0.6738494570739141, + "pass@4": 0.7523035856405045, + "pass@8": 0.8063191188545652, + "pass@16": 0.843643593454054, + "pass@32": 0.8708784076063162, + "pass@64": 0.8951979223494801, + "pass@128": 0.9225806451612903 + }, + "17": { + "pass@1": 0.49896134393063585, + "pass@2": 0.6055844920577128, + "pass@4": 0.6884978144892292, + "pass@8": 0.7497283063889675, + "pass@16": 0.7943429447842064, + "pass@32": 0.8296028860318351, + "pass@64": 0.8601444837620584, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4401785714285714, + "pass@2": 0.5414805961754777, + "pass@4": 0.620062789919117, + "pass@8": 0.6825087231452688, + "pass@16": 0.7330450275756425, + "pass@32": 0.7715735323836825, + "pass@64": 0.8007257492775395, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.3295091966966967, + "pass@2": 0.42963078728334647, + "pass@4": 0.5176115941588382, + "pass@8": 0.5916116555282213, + "pass@16": 0.6522660774252571, + "pass@32": 0.7026441292013839, + "pass@64": 0.7490627189135609, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.3669294294294294, + "pass@2": 0.47350980409543403, + "pass@4": 0.5639906050595023, + "pass@8": 0.6350813513267202, + "pass@16": 0.6886366357150525, + "pass@32": 0.7318928034540644, + "pass@64": 0.770224695930882, + "pass@128": 0.8048048048048048 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 262409, + "answer_accuracy": 0.6095956920903954, + "avg_response_len": 367.53131039994054, + "resp_tokens_sum": 158208998, + "pass_at_k": { + "pass@1": 0.6095956920903954, + "pass@2": 0.6929929922898841, + "pass@4": 0.7563079705079249, + "pass@8": 0.8046729227247553, + "pass@16": 0.8414040625147604, + "pass@32": 0.8690179606347247, + "pass@64": 0.8904732775383289, + "pass@128": 0.9093071662206363 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9468826338819848, + "pass@4": 0.9644491272175316, + "pass@8": 0.9749877614946728, + "pass@16": 0.9813777882066246, + "pass@32": 0.986807323784196, + "pass@64": 0.9916378206179335, + "pass@128": 0.9939759036144579 + }, + "12": { + "pass@1": 0.8360844435736677, + "pass@2": 0.8800076518648338, + "pass@4": 0.9078132809809426, + "pass@8": 0.9269145444634191, + "pass@16": 0.9396337022070549, + "pass@32": 0.9486619447609619, + "pass@64": 0.9548071381739272, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7287289915966386, + "pass@2": 0.7963580609409118, + "pass@4": 0.841953924351893, + "pass@8": 0.876928300220035, + "pass@16": 0.904554157815763, + "pass@32": 0.9254462571555292, + "pass@64": 0.9401177901808709, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7159173976608187, + "pass@2": 0.7988492597964728, + "pass@4": 0.856491532308461, + "pass@8": 0.8950593406393607, + "pass@16": 0.9211022424117776, + "pass@32": 0.9385845730381236, + "pass@64": 0.9499761266570184, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6883049242424243, + "pass@2": 0.7739773472918154, + "pass@4": 0.8314353032575473, + "pass@8": 0.8705559369310063, + "pass@16": 0.8982743454273422, + "pass@32": 0.9179786489223545, + "pass@64": 0.9320109430910483, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5450498575498576, + "pass@2": 0.6449591437288287, + "pass@4": 0.7198456640569499, + "pass@8": 0.7786853841035807, + "pass@16": 0.8259551044670943, + "pass@32": 0.8620595238262503, + "pass@64": 0.88890031356281, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5208171583850931, + "pass@2": 0.6292667475668802, + "pass@4": 0.7093731415366554, + "pass@8": 0.7695169309117884, + "pass@16": 0.8136355479127603, + "pass@32": 0.8447643060918684, + "pass@64": 0.8701041757502938, + "pass@128": 0.8944099378881988 + }, + "18": { + "pass@1": 0.48566176470588235, + "pass@2": 0.5931659622510422, + "pass@4": 0.6829272411720586, + "pass@8": 0.7536475960707826, + "pass@16": 0.8045954899897602, + "pass@32": 0.8405545218075591, + "pass@64": 0.8695012729380446, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.32163783482142855, + "pass@2": 0.41333932379546307, + "pass@4": 0.49753887851072176, + "pass@8": 0.5725680674156789, + "pass@16": 0.6380391354389646, + "pass@32": 0.6926487886121019, + "pass@64": 0.7373374929774971, + "pass@128": 0.7738095238095238 + }, + "20": { + "pass@1": 0.3458317739520958, + "pass@2": 0.4579825339478522, + "pass@4": 0.5549417604610802, + "pass@8": 0.6303896133806527, + "pass@16": 0.6882139105165509, + "pass@32": 0.733040383229659, + "pass@64": 0.7701302877722762, + "pass@128": 0.8053892215568862 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 253272, + "answer_accuracy": 0.6047333435207825, + "avg_response_len": 396.2800585459963, + "resp_tokens_sum": 165968429, + "pass_at_k": { + "pass@1": 0.6047333435207825, + "pass@2": 0.6889823189602102, + "pass@4": 0.7528605627001419, + "pass@8": 0.8014700923532283, + "pass@16": 0.8387872359644982, + "pass@32": 0.8683080825213123, + "pass@64": 0.8922519179719953, + "pass@128": 0.910757946210269 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8957093253968254, + "pass@2": 0.9324045431821019, + "pass@4": 0.9530571291683771, + "pass@8": 0.9661005237823533, + "pass@16": 0.9753324158172761, + "pass@32": 0.9817093737531953, + "pass@64": 0.9852794285216058, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8548595183486238, + "pass@2": 0.9026862252883529, + "pass@4": 0.928716852767256, + "pass@8": 0.9458145843657703, + "pass@16": 0.9596806462958717, + "pass@32": 0.9727740288131387, + "pass@64": 0.9845103741794053, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.728515625, + "pass@2": 0.8026403926071737, + "pass@4": 0.854766033875395, + "pass@8": 0.8928687155866702, + "pass@16": 0.920794415945995, + "pass@32": 0.94152406634448, + "pass@64": 0.9565434779815419, + "pass@128": 0.9660493827160493 + }, + "14": { + "pass@1": 0.7207205118694362, + "pass@2": 0.7957049230122201, + "pass@4": 0.8462405605271146, + "pass@8": 0.8793643436005479, + "pass@16": 0.9014542133541928, + "pass@32": 0.9174294131525637, + "pass@64": 0.9303251689752986, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6839554398148148, + "pass@2": 0.7633861305531252, + "pass@4": 0.8194027031343298, + "pass@8": 0.8602466462398325, + "pass@16": 0.8887469421251796, + "pass@32": 0.9087115703307098, + "pass@64": 0.9239886621606587, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5756130162241888, + "pass@2": 0.6845900692170115, + "pass@4": 0.761188147112142, + "pass@8": 0.8149786242373147, + "pass@16": 0.8555630664100949, + "pass@32": 0.8888727828261448, + "pass@64": 0.9169712192641652, + "pass@128": 0.9351032448377581 + }, + "17": { + "pass@1": 0.4842808734939759, + "pass@2": 0.584871350559719, + "pass@4": 0.6645132360149255, + "pass@8": 0.7297835726231425, + "pass@16": 0.785105968775507, + "pass@32": 0.831487801164966, + "pass@64": 0.8694257138315328, + "pass@128": 0.9006024096385542 + }, + "18": { + "pass@1": 0.4754536290322581, + "pass@2": 0.5808114522479044, + "pass@4": 0.6585383135575791, + "pass@8": 0.7148880869953559, + "pass@16": 0.7581914940782433, + "pass@32": 0.7942661179834702, + "pass@64": 0.8252641405234724, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.325693919939577, + "pass@2": 0.4348405125246804, + "pass@4": 0.5347291311962133, + "pass@8": 0.6173469310391171, + "pass@16": 0.6816601314840767, + "pass@32": 0.7327839098583034, + "pass@64": 0.7748918502096696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3164179804804805, + "pass@2": 0.4194657255680876, + "pass@4": 0.5166342914905905, + "pass@8": 0.6003871057227763, + "pass@16": 0.6667422102319082, + "pass@32": 0.7176055052180353, + "pass@64": 0.7583024072323226, + "pass@128": 0.7867867867867868 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7824979440789473, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7824979440789473, + "pass@2": 0.8308864419291083, + "pass@4": 0.8662580142926025, + "pass@8": 0.8927520757656101, + "pass@16": 0.9128875901187812, + "pass@32": 0.9285626010725488, + "pass@64": 0.9412129591444078, + "pass@128": 0.9517894736842105 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.91640625, + "pass@2": 0.9476706446850395, + "pass@4": 0.9669583255530562, + "pass@8": 0.9788451880131132, + "pass@16": 0.9862062777304876, + "pass@32": 0.9912431276236333, + "pass@64": 0.9947722680345038, + "pass@128": 0.997 + }, + "2": { + "pass@1": 0.99846875, + "pass@2": 0.9988816437007875, + "pass@4": 0.9989923884514436, + "pass@8": 0.9989999655709787, + "pass@16": 0.9989999999997617, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.9932734375, + "pass@2": 0.9957941683070867, + "pass@4": 0.9978264966254219, + "pass@8": 0.9990544210606774, + "pass@16": 0.9996775186156238, + "pass@32": 0.9999476963247053, + "pass@64": 0.9999993318674025, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992609375, + "pass@2": 0.9941601870078742, + "pass@4": 0.995101639295088, + "pass@8": 0.9959862454184044, + "pass@16": 0.9967492275545933, + "pass@32": 0.9973881759502105, + "pass@64": 0.9979968420843298, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9737109375, + "pass@2": 0.9815353100393699, + "pass@4": 0.9869551624484438, + "pass@8": 0.9911598043008092, + "pass@16": 0.9944993273291891, + "pass@32": 0.9971205319942156, + "pass@64": 0.9989734239317924, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9908984375, + "pass@2": 0.9944271653543306, + "pass@4": 0.9961795943007125, + "pass@8": 0.9970593753784408, + "pass@16": 0.9976541287314281, + "pass@32": 0.9981434924936788, + "pass@64": 0.9984968766172793, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.983296875, + "pass@2": 0.9901711368110238, + "pass@4": 0.9935972257217848, + "pass@8": 0.9958620196079306, + "pass@16": 0.997439017820933, + "pass@32": 0.9986513545851102, + "pass@64": 0.9995945257898196, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.974046875, + "pass@2": 0.9852920767716542, + "pass@4": 0.9903052625609297, + "pass@8": 0.9934548398646773, + "pass@16": 0.9956272175270217, + "pass@32": 0.9970991849627101, + "pass@64": 0.997866112633597, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.96434375, + "pass@2": 0.9808385826771662, + "pass@4": 0.9870510364641919, + "pass@8": 0.9907272684654614, + "pass@16": 0.993351928589844, + "pass@32": 0.9953961168156369, + "pass@64": 0.9972384335631338, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.89928125, + "pass@2": 0.9340199311023635, + "pass@4": 0.9530581857892761, + "pass@8": 0.9646084167445874, + "pass@16": 0.9721545862699154, + "pass@32": 0.978247602678375, + "pass@64": 0.9833447019063941, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.84053125, + "pass@2": 0.8881071604330706, + "pass@4": 0.9157314617547808, + "pass@8": 0.9342534388562534, + "pass@16": 0.9479341431792865, + "pass@32": 0.9595398724244704, + "pass@64": 0.9696723257993726, + "pass@128": 0.978 + }, + "13": { + "pass@1": 0.7338671875, + "pass@2": 0.8052672244094486, + "pass@4": 0.8534425268091481, + "pass@8": 0.8896280507676896, + "pass@16": 0.9178852960128545, + "pass@32": 0.9396084313296421, + "pass@64": 0.9551960359821134, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.7240703125, + "pass@2": 0.8020214074803148, + "pass@4": 0.8532807261904767, + "pass@8": 0.8874175863692658, + "pass@16": 0.9118375285344886, + "pass@32": 0.9301418992032527, + "pass@64": 0.9433938844205266, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.6850625, + "pass@2": 0.7675178395669306, + "pass@4": 0.8241469076677913, + "pass@8": 0.8637729210547304, + "pass@16": 0.8922964442166276, + "pass@32": 0.9133260490070577, + "pass@64": 0.9290881232929966, + "pass@128": 0.942 + }, + "16": { + "pass@1": 0.5612265625, + "pass@2": 0.667350024606298, + "pass@4": 0.743922721503563, + "pass@8": 0.799555250281722, + "pass@16": 0.8414756351517275, + "pass@32": 0.8738830725990342, + "pass@64": 0.9003686093194375, + "pass@128": 0.923 + }, + "17": { + "pass@1": 0.501125, + "pass@2": 0.6063334153543315, + "pass@4": 0.6872567897450311, + "pass@8": 0.7494785918750635, + "pass@16": 0.7974884869567112, + "pass@32": 0.8351106551153658, + "pass@64": 0.8664328729653366, + "pass@128": 0.893 + }, + "18": { + "pass@1": 0.466578125, + "pass@2": 0.571246186023622, + "pass@4": 0.6533641156730406, + "pass@8": 0.7167335427334711, + "pass@16": 0.7651675894122477, + "pass@32": 0.8020617703237342, + "pass@64": 0.831716328608351, + "pass@128": 0.858 + }, + "19": { + "pass@1": 0.3256015625, + "pass@2": 0.42588127460629893, + "pass@4": 0.5165330664604426, + "pass@8": 0.5937313861165137, + "pass@16": 0.6572152568113311, + "pass@32": 0.7092619621608254, + "pass@64": 0.7536724854580563, + "pass@128": 0.793 + }, + "20": { + "pass@1": 0.3430625, + "pass@2": 0.450327017716536, + "pass@4": 0.5451986385451821, + "pass@8": 0.6219611270666211, + "pass@16": 0.6812046018128655, + "pass@32": 0.7275184247865151, + "pass@64": 0.7662230414692883, + "pass@128": 0.799 + } + }, + "per_op_accuracy": { + "10": 0.91640625, + "2": 0.99846875, + "3": 0.9932734375, + "4": 0.992609375, + "5": 0.9737109375, + "6": 0.9908984375, + "7": 0.983296875, + "8": 0.974046875, + "9": 0.96434375, + "11": 0.89928125, + "12": 0.84053125, + "13": 0.7338671875, + "14": 0.7240703125, + "15": 0.6850625, + "16": 0.5612265625, + "17": 0.501125, + "18": 0.466578125, + "19": 0.3256015625, + "20": 0.3430625 + }, + "per_op_avg_response_len": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375, + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375, + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 631866, + "answer_accuracy": 0.7803435227631995, + "avg_response_len": 277.16480966447995, + "resp_tokens_sum": 224428107, + "pass_at_k": { + "pass@1": 0.7803435227631995, + "pass@2": 0.8294686416327026, + "pass@4": 0.8645065197450194, + "pass@8": 0.8903853309345585, + "pass@16": 0.9100911579608634, + "pass@32": 0.9257940995293442, + "pass@64": 0.9389749729084804, + "pass@128": 0.9505216566550743 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9147999631268436, + "pass@2": 0.9457115357814785, + "pass@4": 0.9644939742045515, + "pass@8": 0.976467977625549, + "pass@16": 0.9845557191374704, + "pass@32": 0.9904786548196389, + "pass@64": 0.9943848945020379, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9949589067278287, + "pass@2": 0.9971454435454742, + "pass@4": 0.998757460363326, + "pass@8": 0.9996935826438439, + "pass@16": 0.9999781984079241, + "pass@32": 0.9999999385362756, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985107421875, + "pass@2": 0.9878137303149606, + "pass@4": 0.9902266852385638, + "pass@8": 0.992418635965382, + "pass@16": 0.9940056498208453, + "pass@32": 0.9952528850596998, + "pass@64": 0.9968748060893958, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9661954365079365, + "pass@2": 0.9754136201724785, + "pass@4": 0.9818137741710856, + "pass@8": 0.9867755777021514, + "pass@16": 0.9913209124055069, + "pass@32": 0.9954008763900372, + "pass@64": 0.9983700744266549, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9920471556886228, + "pass@2": 0.9955804434438211, + "pass@4": 0.9972394134677776, + "pass@8": 0.998204156494888, + "pass@16": 0.999004031075878, + "pass@32": 0.9997223135643741, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9807923446745562, + "pass@2": 0.9899740250663933, + "pass@4": 0.9942786351040437, + "pass@8": 0.9966196608845793, + "pass@16": 0.9978572891506724, + "pass@32": 0.9987563633971823, + "pass@64": 0.9996389129939015, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9767878605769231, + "pass@2": 0.9883080645568344, + "pass@4": 0.9921772134252452, + "pass@8": 0.9944861816978635, + "pass@16": 0.9959239960690798, + "pass@32": 0.9966348318465099, + "pass@64": 0.9967927316152657, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9637764084507042, + "pass@2": 0.9817663857158702, + "pass@4": 0.9897242576192058, + "pass@8": 0.994632208919704, + "pass@16": 0.9970542065127949, + "pass@32": 0.9978697672017515, + "pass@64": 0.9985915429355597, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8884782223796034, + "pass@2": 0.9233639250741679, + "pass@4": 0.9423458353116624, + "pass@8": 0.9535150564785112, + "pass@16": 0.9606443331526207, + "pass@32": 0.9671079842769559, + "pass@64": 0.9738184857703525, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.831302966101695, + "pass@2": 0.8819387678499936, + "pass@4": 0.9108717917145948, + "pass@8": 0.9301873731209482, + "pass@16": 0.9445633356962888, + "pass@32": 0.9571175271859526, + "pass@64": 0.9693614586588243, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7450528996865203, + "pass@2": 0.817905672253351, + "pass@4": 0.8649554258303918, + "pass@8": 0.900549165326192, + "pass@16": 0.9298497520568195, + "pass@32": 0.9535119750141245, + "pass@64": 0.9707018746756205, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.7362733644859814, + "pass@2": 0.8120324036598229, + "pass@4": 0.8572509446833161, + "pass@8": 0.8877305547576959, + "pass@16": 0.9128675754806822, + "pass@32": 0.9344930311271058, + "pass@64": 0.9501011437979755, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.6830066835260116, + "pass@2": 0.7652260365936912, + "pass@4": 0.8216380976221894, + "pass@8": 0.8606056314618283, + "pass@16": 0.8899187889510047, + "pass@32": 0.9132096707385248, + "pass@64": 0.9310756807309161, + "pass@128": 0.9479768786127167 + }, + "16": { + "pass@1": 0.5638104838709678, + "pass@2": 0.6738494570739141, + "pass@4": 0.7523035856405045, + "pass@8": 0.8063191188545652, + "pass@16": 0.843643593454054, + "pass@32": 0.8708784076063162, + "pass@64": 0.8951979223494801, + "pass@128": 0.9225806451612903 + }, + "17": { + "pass@1": 0.49896134393063585, + "pass@2": 0.6055844920577128, + "pass@4": 0.6884978144892292, + "pass@8": 0.7497283063889675, + "pass@16": 0.7943429447842064, + "pass@32": 0.8296028860318351, + "pass@64": 0.8601444837620584, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4401785714285714, + "pass@2": 0.5414805961754777, + "pass@4": 0.620062789919117, + "pass@8": 0.6825087231452688, + "pass@16": 0.7330450275756425, + "pass@32": 0.7715735323836825, + "pass@64": 0.8007257492775395, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.3295091966966967, + "pass@2": 0.42963078728334647, + "pass@4": 0.5176115941588382, + "pass@8": 0.5916116555282213, + "pass@16": 0.6522660774252571, + "pass@32": 0.7026441292013839, + "pass@64": 0.7490627189135609, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.3669294294294294, + "pass@2": 0.47350980409543403, + "pass@4": 0.5639906050595023, + "pass@8": 0.6350813513267202, + "pass@16": 0.6886366357150525, + "pass@32": 0.7318928034540644, + "pass@64": 0.770224695930882, + "pass@128": 0.8048048048048048 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644522, + "answer_accuracy": 0.7849303390491037, + "avg_response_len": 286.6854004286828, + "resp_tokens_sum": 235403116, + "pass_at_k": { + "pass@1": 0.7849303390491037, + "pass@2": 0.8327475896183206, + "pass@4": 0.8680801569425771, + "pass@8": 0.8946791462011701, + "pass@16": 0.9147568033503521, + "pass@32": 0.9298668825918631, + "pass@64": 0.9416667394710053, + "pass@128": 0.951987529228371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.915763995398773, + "pass@2": 0.9480459881165166, + "pass@4": 0.9681581655245539, + "pass@8": 0.9794301390879024, + "pass@16": 0.9853206794298328, + "pass@32": 0.989459399841, + "pass@64": 0.9934911833242681, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9990976443768997, + "pass@2": 0.9997371089916952, + "pass@4": 0.9999789687193356, + "pass@8": 0.9999998960323445, + "pass@16": 0.9999999999992757, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990814393939394, + "pass@2": 0.9942462270341206, + "pass@4": 0.9969956352046904, + "pass@8": 0.9983947582818798, + "pass@16": 0.9992437871465215, + "pass@32": 0.9998493181618995, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959781804733728, + "pass@2": 0.9972641872058892, + "pass@4": 0.9977194152506086, + "pass@8": 0.9982611890819908, + "pass@16": 0.9990160343548016, + "pass@32": 0.9997255997961729, + "pass@64": 0.9999908406382343, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9807065217391304, + "pass@2": 0.9866305774278213, + "pass@4": 0.9910699586464737, + "pass@8": 0.9945806050238211, + "pass@16": 0.997016986481387, + "pass@32": 0.9983197945450435, + "pass@64": 0.9992809854172511, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867030802292264, + "pass@2": 0.9902159995938904, + "pass@4": 0.9921985556783911, + "pass@8": 0.9933309007238308, + "pass@16": 0.9942316250648926, + "pass@32": 0.9949462457401639, + "pass@64": 0.9957019268812495, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9899972904624278, + "pass@2": 0.9941499590369122, + "pass@4": 0.9955985682425533, + "pass@8": 0.9965680962120179, + "pass@16": 0.9975371405733434, + "pass@32": 0.9983638978272487, + "pass@64": 0.9992831394824679, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9800836267605634, + "pass@2": 0.9910738050349339, + "pass@4": 0.9956219548260692, + "pass@8": 0.997544552113714, + "pass@16": 0.9983496167020315, + "pass@32": 0.9991228206283554, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9617795658682635, + "pass@2": 0.9778661181573863, + "pass@4": 0.9829365481635154, + "pass@8": 0.98602176677128, + "pass@16": 0.9891346058079603, + "pass@32": 0.9921804769068225, + "pass@64": 0.9949455674393275, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9468826338819848, + "pass@4": 0.9644491272175316, + "pass@8": 0.9749877614946728, + "pass@16": 0.9813777882066246, + "pass@32": 0.986807323784196, + "pass@64": 0.9916378206179335, + "pass@128": 0.9939759036144579 + }, + "12": { + "pass@1": 0.8360844435736677, + "pass@2": 0.8800076518648338, + "pass@4": 0.9078132809809426, + "pass@8": 0.9269145444634191, + "pass@16": 0.9396337022070549, + "pass@32": 0.9486619447609619, + "pass@64": 0.9548071381739272, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7287289915966386, + "pass@2": 0.7963580609409118, + "pass@4": 0.841953924351893, + "pass@8": 0.876928300220035, + "pass@16": 0.904554157815763, + "pass@32": 0.9254462571555292, + "pass@64": 0.9401177901808709, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7159173976608187, + "pass@2": 0.7988492597964728, + "pass@4": 0.856491532308461, + "pass@8": 0.8950593406393607, + "pass@16": 0.9211022424117776, + "pass@32": 0.9385845730381236, + "pass@64": 0.9499761266570184, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6883049242424243, + "pass@2": 0.7739773472918154, + "pass@4": 0.8314353032575473, + "pass@8": 0.8705559369310063, + "pass@16": 0.8982743454273422, + "pass@32": 0.9179786489223545, + "pass@64": 0.9320109430910483, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5450498575498576, + "pass@2": 0.6449591437288287, + "pass@4": 0.7198456640569499, + "pass@8": 0.7786853841035807, + "pass@16": 0.8259551044670943, + "pass@32": 0.8620595238262503, + "pass@64": 0.88890031356281, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5208171583850931, + "pass@2": 0.6292667475668802, + "pass@4": 0.7093731415366554, + "pass@8": 0.7695169309117884, + "pass@16": 0.8136355479127603, + "pass@32": 0.8447643060918684, + "pass@64": 0.8701041757502938, + "pass@128": 0.8944099378881988 + }, + "18": { + "pass@1": 0.48566176470588235, + "pass@2": 0.5931659622510422, + "pass@4": 0.6829272411720586, + "pass@8": 0.7536475960707826, + "pass@16": 0.8045954899897602, + "pass@32": 0.8405545218075591, + "pass@64": 0.8695012729380446, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.32163783482142855, + "pass@2": 0.41333932379546307, + "pass@4": 0.49753887851072176, + "pass@8": 0.5725680674156789, + "pass@16": 0.6380391354389646, + "pass@32": 0.6926487886121019, + "pass@64": 0.7373374929774971, + "pass@128": 0.7738095238095238 + }, + "20": { + "pass@1": 0.3458317739520958, + "pass@2": 0.4579825339478522, + "pass@4": 0.5549417604610802, + "pass@8": 0.6303896133806527, + "pass@16": 0.6882139105165509, + "pass@32": 0.733040383229659, + "pass@64": 0.7701302877722762, + "pass@128": 0.8053892215568862 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 626647, + "answer_accuracy": 0.7821824073334398, + "avg_response_len": 307.9202261243809, + "resp_tokens_sum": 246690905, + "pass_at_k": { + "pass@1": 0.7821824073334398, + "pass@2": 0.8304118840523114, + "pass@4": 0.8661606999306755, + "pass@8": 0.8931690546690862, + "pass@16": 0.9137981552171506, + "pass@32": 0.9300239487024705, + "pass@64": 0.9430098116980766, + "pass@128": 0.9528678702668157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9186567164179105, + "pass@2": 0.9492878863556237, + "pass@4": 0.9682844965274856, + "pass@8": 0.9806815470310305, + "pass@16": 0.988738350593314, + "pass@32": 0.9937525351988347, + "pass@64": 0.9964109344316464, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9993303571428571, + "pass@2": 0.9999089566929135, + "pass@4": 0.9999980221222345, + "pass@8": 0.9999999993609641, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940324344023324, + "pass@2": 0.9959951992608066, + "pass@4": 0.9977383308136045, + "pass@8": 0.9990797355776098, + "pass@16": 0.9998081573699138, + "pass@32": 0.999992540612001, + "pass@64": 0.9999999988420875, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962993421052632, + "pass@2": 0.9970306959985263, + "pass@4": 0.9970758411229297, + "pass@8": 0.9970760233911381, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9735753676470589, + "pass@2": 0.9820366778601207, + "pass@4": 0.9875431995632898, + "pass@8": 0.9917505546806278, + "pass@16": 0.994889351721694, + "pass@32": 0.9974968435685705, + "pass@64": 0.9992203250542475, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9943069794952681, + "pass@2": 0.9978483072108101, + "pass@4": 0.9994458494344357, + "pass@8": 0.9999580528597197, + "pass@16": 0.9999998271433355, + "pass@32": 0.999999999999561, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9786392405063291, + "pass@2": 0.9860254130120601, + "pass@4": 0.9906770330528303, + "pass@8": 0.9942785219607105, + "pass@16": 0.9968841881317372, + "pass@32": 0.9988537819893496, + "pass@64": 0.9998879966359081, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9650431681681682, + "pass@2": 0.9763025821096688, + "pass@4": 0.982883417444441, + "pass@8": 0.9881286341578839, + "pass@16": 0.9924468973701125, + "pass@32": 0.9953769252358343, + "pass@64": 0.996775794068325, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9677451768488746, + "pass@2": 0.9829718079854168, + "pass@4": 0.9884183856037287, + "pass@8": 0.9913233575477782, + "pass@16": 0.9936550705400102, + "pass@32": 0.9960259491065468, + "pass@64": 0.998156322496061, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8957093253968254, + "pass@2": 0.9324045431821019, + "pass@4": 0.9530571291683771, + "pass@8": 0.9661005237823533, + "pass@16": 0.9753324158172761, + "pass@32": 0.9817093737531953, + "pass@64": 0.9852794285216058, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8548595183486238, + "pass@2": 0.9026862252883529, + "pass@4": 0.928716852767256, + "pass@8": 0.9458145843657703, + "pass@16": 0.9596806462958717, + "pass@32": 0.9727740288131387, + "pass@64": 0.9845103741794053, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.728515625, + "pass@2": 0.8026403926071737, + "pass@4": 0.854766033875395, + "pass@8": 0.8928687155866702, + "pass@16": 0.920794415945995, + "pass@32": 0.94152406634448, + "pass@64": 0.9565434779815419, + "pass@128": 0.9660493827160493 + }, + "14": { + "pass@1": 0.7207205118694362, + "pass@2": 0.7957049230122201, + "pass@4": 0.8462405605271146, + "pass@8": 0.8793643436005479, + "pass@16": 0.9014542133541928, + "pass@32": 0.9174294131525637, + "pass@64": 0.9303251689752986, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6839554398148148, + "pass@2": 0.7633861305531252, + "pass@4": 0.8194027031343298, + "pass@8": 0.8602466462398325, + "pass@16": 0.8887469421251796, + "pass@32": 0.9087115703307098, + "pass@64": 0.9239886621606587, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5756130162241888, + "pass@2": 0.6845900692170115, + "pass@4": 0.761188147112142, + "pass@8": 0.8149786242373147, + "pass@16": 0.8555630664100949, + "pass@32": 0.8888727828261448, + "pass@64": 0.9169712192641652, + "pass@128": 0.9351032448377581 + }, + "17": { + "pass@1": 0.4842808734939759, + "pass@2": 0.584871350559719, + "pass@4": 0.6645132360149255, + "pass@8": 0.7297835726231425, + "pass@16": 0.785105968775507, + "pass@32": 0.831487801164966, + "pass@64": 0.8694257138315328, + "pass@128": 0.9006024096385542 + }, + "18": { + "pass@1": 0.4754536290322581, + "pass@2": 0.5808114522479044, + "pass@4": 0.6585383135575791, + "pass@8": 0.7148880869953559, + "pass@16": 0.7581914940782433, + "pass@32": 0.7942661179834702, + "pass@64": 0.8252641405234724, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.325693919939577, + "pass@2": 0.4348405125246804, + "pass@4": 0.5347291311962133, + "pass@8": 0.6173469310391171, + "pass@16": 0.6816601314840767, + "pass@32": 0.7327839098583034, + "pass@64": 0.7748918502096696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3164179804804805, + "pass@2": 0.4194657255680876, + "pass@4": 0.5166342914905905, + "pass@8": 0.6003871057227763, + "pass@16": 0.6667422102319082, + "pass@32": 0.7176055052180353, + "pass@64": 0.7583024072323226, + "pass@128": 0.7867867867867868 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..008bf0960f3dd43d62c37db0f4cebb3e3c5af662 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14aa5c05aa2da6abbd8bff2b56e377327736e8db31260e7ad5081fbdc3924eb0 +size 2708341168 diff --git a/results/composition/SDAR-100M-block8-epoch1-op17-20/summary.csv b/results/composition/SDAR-100M-block8-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..17e1e7ebd6b2765dd235f1fe1c67639ecd558f2b --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op17-20,0.9763394097222222,0.608040625,0.7824979440789473,0.0,0.0,0.0,199.7977092013889,372.15247421875,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch1-op17-20/summary.json b/results/composition/SDAR-100M-block8-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e6f6d82323f9b89125541c2f9f7489f9fc82b99f --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op17-20", + "id_acc": 0.9763394097222222, + "ood_acc": 0.608040625, + "total_acc": 0.7824979440789473, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.7977092013889, + "ood_avg_resp_len": 372.15247421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 631866, + "answer_accuracy": 0.7803435227631995, + "avg_response_len": 277.16480966447995, + "resp_tokens_sum": 224428107, + "pass_at_k": { + "pass@1": 0.7803435227631995, + "pass@2": 0.8294686416327026, + "pass@4": 0.8645065197450194, + "pass@8": 0.8903853309345585, + "pass@16": 0.9100911579608634, + "pass@32": 0.9257940995293442, + "pass@64": 0.9389749729084804, + "pass@128": 0.9505216566550743 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9147999631268436, + "pass@2": 0.9457115357814785, + "pass@4": 0.9644939742045515, + "pass@8": 0.976467977625549, + "pass@16": 0.9845557191374704, + "pass@32": 0.9904786548196389, + "pass@64": 0.9943848945020379, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9949589067278287, + "pass@2": 0.9971454435454742, + "pass@4": 0.998757460363326, + "pass@8": 0.9996935826438439, + "pass@16": 0.9999781984079241, + "pass@32": 0.9999999385362756, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985107421875, + "pass@2": 0.9878137303149606, + "pass@4": 0.9902266852385638, + "pass@8": 0.992418635965382, + "pass@16": 0.9940056498208453, + "pass@32": 0.9952528850596998, + "pass@64": 0.9968748060893958, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9661954365079365, + "pass@2": 0.9754136201724785, + "pass@4": 0.9818137741710856, + "pass@8": 0.9867755777021514, + "pass@16": 0.9913209124055069, + "pass@32": 0.9954008763900372, + "pass@64": 0.9983700744266549, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9920471556886228, + "pass@2": 0.9955804434438211, + "pass@4": 0.9972394134677776, + "pass@8": 0.998204156494888, + "pass@16": 0.999004031075878, + "pass@32": 0.9997223135643741, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9807923446745562, + "pass@2": 0.9899740250663933, + "pass@4": 0.9942786351040437, + "pass@8": 0.9966196608845793, + "pass@16": 0.9978572891506724, + "pass@32": 0.9987563633971823, + "pass@64": 0.9996389129939015, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9767878605769231, + "pass@2": 0.9883080645568344, + "pass@4": 0.9921772134252452, + "pass@8": 0.9944861816978635, + "pass@16": 0.9959239960690798, + "pass@32": 0.9966348318465099, + "pass@64": 0.9967927316152657, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9637764084507042, + "pass@2": 0.9817663857158702, + "pass@4": 0.9897242576192058, + "pass@8": 0.994632208919704, + "pass@16": 0.9970542065127949, + "pass@32": 0.9978697672017515, + "pass@64": 0.9985915429355597, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8884782223796034, + "pass@2": 0.9233639250741679, + "pass@4": 0.9423458353116624, + "pass@8": 0.9535150564785112, + "pass@16": 0.9606443331526207, + "pass@32": 0.9671079842769559, + "pass@64": 0.9738184857703525, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.831302966101695, + "pass@2": 0.8819387678499936, + "pass@4": 0.9108717917145948, + "pass@8": 0.9301873731209482, + "pass@16": 0.9445633356962888, + "pass@32": 0.9571175271859526, + "pass@64": 0.9693614586588243, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7450528996865203, + "pass@2": 0.817905672253351, + "pass@4": 0.8649554258303918, + "pass@8": 0.900549165326192, + "pass@16": 0.9298497520568195, + "pass@32": 0.9535119750141245, + "pass@64": 0.9707018746756205, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.7362733644859814, + "pass@2": 0.8120324036598229, + "pass@4": 0.8572509446833161, + "pass@8": 0.8877305547576959, + "pass@16": 0.9128675754806822, + "pass@32": 0.9344930311271058, + "pass@64": 0.9501011437979755, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.6830066835260116, + "pass@2": 0.7652260365936912, + "pass@4": 0.8216380976221894, + "pass@8": 0.8606056314618283, + "pass@16": 0.8899187889510047, + "pass@32": 0.9132096707385248, + "pass@64": 0.9310756807309161, + "pass@128": 0.9479768786127167 + }, + "16": { + "pass@1": 0.5638104838709678, + "pass@2": 0.6738494570739141, + "pass@4": 0.7523035856405045, + "pass@8": 0.8063191188545652, + "pass@16": 0.843643593454054, + "pass@32": 0.8708784076063162, + "pass@64": 0.8951979223494801, + "pass@128": 0.9225806451612903 + }, + "17": { + "pass@1": 0.49896134393063585, + "pass@2": 0.6055844920577128, + "pass@4": 0.6884978144892292, + "pass@8": 0.7497283063889675, + "pass@16": 0.7943429447842064, + "pass@32": 0.8296028860318351, + "pass@64": 0.8601444837620584, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4401785714285714, + "pass@2": 0.5414805961754777, + "pass@4": 0.620062789919117, + "pass@8": 0.6825087231452688, + "pass@16": 0.7330450275756425, + "pass@32": 0.7715735323836825, + "pass@64": 0.8007257492775395, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.3295091966966967, + "pass@2": 0.42963078728334647, + "pass@4": 0.5176115941588382, + "pass@8": 0.5916116555282213, + "pass@16": 0.6522660774252571, + "pass@32": 0.7026441292013839, + "pass@64": 0.7490627189135609, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.3669294294294294, + "pass@2": 0.47350980409543403, + "pass@4": 0.5639906050595023, + "pass@8": 0.6350813513267202, + "pass@16": 0.6886366357150525, + "pass@32": 0.7318928034540644, + "pass@64": 0.770224695930882, + "pass@128": 0.8048048048048048 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644522, + "answer_accuracy": 0.7849303390491037, + "avg_response_len": 286.6854004286828, + "resp_tokens_sum": 235403116, + "pass_at_k": { + "pass@1": 0.7849303390491037, + "pass@2": 0.8327475896183206, + "pass@4": 0.8680801569425771, + "pass@8": 0.8946791462011701, + "pass@16": 0.9147568033503521, + "pass@32": 0.9298668825918631, + "pass@64": 0.9416667394710053, + "pass@128": 0.951987529228371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.915763995398773, + "pass@2": 0.9480459881165166, + "pass@4": 0.9681581655245539, + "pass@8": 0.9794301390879024, + "pass@16": 0.9853206794298328, + "pass@32": 0.989459399841, + "pass@64": 0.9934911833242681, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9990976443768997, + "pass@2": 0.9997371089916952, + "pass@4": 0.9999789687193356, + "pass@8": 0.9999998960323445, + "pass@16": 0.9999999999992757, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990814393939394, + "pass@2": 0.9942462270341206, + "pass@4": 0.9969956352046904, + "pass@8": 0.9983947582818798, + "pass@16": 0.9992437871465215, + "pass@32": 0.9998493181618995, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959781804733728, + "pass@2": 0.9972641872058892, + "pass@4": 0.9977194152506086, + "pass@8": 0.9982611890819908, + "pass@16": 0.9990160343548016, + "pass@32": 0.9997255997961729, + "pass@64": 0.9999908406382343, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9807065217391304, + "pass@2": 0.9866305774278213, + "pass@4": 0.9910699586464737, + "pass@8": 0.9945806050238211, + "pass@16": 0.997016986481387, + "pass@32": 0.9983197945450435, + "pass@64": 0.9992809854172511, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867030802292264, + "pass@2": 0.9902159995938904, + "pass@4": 0.9921985556783911, + "pass@8": 0.9933309007238308, + "pass@16": 0.9942316250648926, + "pass@32": 0.9949462457401639, + "pass@64": 0.9957019268812495, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9899972904624278, + "pass@2": 0.9941499590369122, + "pass@4": 0.9955985682425533, + "pass@8": 0.9965680962120179, + "pass@16": 0.9975371405733434, + "pass@32": 0.9983638978272487, + "pass@64": 0.9992831394824679, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9800836267605634, + "pass@2": 0.9910738050349339, + "pass@4": 0.9956219548260692, + "pass@8": 0.997544552113714, + "pass@16": 0.9983496167020315, + "pass@32": 0.9991228206283554, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9617795658682635, + "pass@2": 0.9778661181573863, + "pass@4": 0.9829365481635154, + "pass@8": 0.98602176677128, + "pass@16": 0.9891346058079603, + "pass@32": 0.9921804769068225, + "pass@64": 0.9949455674393275, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9468826338819848, + "pass@4": 0.9644491272175316, + "pass@8": 0.9749877614946728, + "pass@16": 0.9813777882066246, + "pass@32": 0.986807323784196, + "pass@64": 0.9916378206179335, + "pass@128": 0.9939759036144579 + }, + "12": { + "pass@1": 0.8360844435736677, + "pass@2": 0.8800076518648338, + "pass@4": 0.9078132809809426, + "pass@8": 0.9269145444634191, + "pass@16": 0.9396337022070549, + "pass@32": 0.9486619447609619, + "pass@64": 0.9548071381739272, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7287289915966386, + "pass@2": 0.7963580609409118, + "pass@4": 0.841953924351893, + "pass@8": 0.876928300220035, + "pass@16": 0.904554157815763, + "pass@32": 0.9254462571555292, + "pass@64": 0.9401177901808709, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7159173976608187, + "pass@2": 0.7988492597964728, + "pass@4": 0.856491532308461, + "pass@8": 0.8950593406393607, + "pass@16": 0.9211022424117776, + "pass@32": 0.9385845730381236, + "pass@64": 0.9499761266570184, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6883049242424243, + "pass@2": 0.7739773472918154, + "pass@4": 0.8314353032575473, + "pass@8": 0.8705559369310063, + "pass@16": 0.8982743454273422, + "pass@32": 0.9179786489223545, + "pass@64": 0.9320109430910483, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5450498575498576, + "pass@2": 0.6449591437288287, + "pass@4": 0.7198456640569499, + "pass@8": 0.7786853841035807, + "pass@16": 0.8259551044670943, + "pass@32": 0.8620595238262503, + "pass@64": 0.88890031356281, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5208171583850931, + "pass@2": 0.6292667475668802, + "pass@4": 0.7093731415366554, + "pass@8": 0.7695169309117884, + "pass@16": 0.8136355479127603, + "pass@32": 0.8447643060918684, + "pass@64": 0.8701041757502938, + "pass@128": 0.8944099378881988 + }, + "18": { + "pass@1": 0.48566176470588235, + "pass@2": 0.5931659622510422, + "pass@4": 0.6829272411720586, + "pass@8": 0.7536475960707826, + "pass@16": 0.8045954899897602, + "pass@32": 0.8405545218075591, + "pass@64": 0.8695012729380446, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.32163783482142855, + "pass@2": 0.41333932379546307, + "pass@4": 0.49753887851072176, + "pass@8": 0.5725680674156789, + "pass@16": 0.6380391354389646, + "pass@32": 0.6926487886121019, + "pass@64": 0.7373374929774971, + "pass@128": 0.7738095238095238 + }, + "20": { + "pass@1": 0.3458317739520958, + "pass@2": 0.4579825339478522, + "pass@4": 0.5549417604610802, + "pass@8": 0.6303896133806527, + "pass@16": 0.6882139105165509, + "pass@32": 0.733040383229659, + "pass@64": 0.7701302877722762, + "pass@128": 0.8053892215568862 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 626647, + "answer_accuracy": 0.7821824073334398, + "avg_response_len": 307.9202261243809, + "resp_tokens_sum": 246690905, + "pass_at_k": { + "pass@1": 0.7821824073334398, + "pass@2": 0.8304118840523114, + "pass@4": 0.8661606999306755, + "pass@8": 0.8931690546690862, + "pass@16": 0.9137981552171506, + "pass@32": 0.9300239487024705, + "pass@64": 0.9430098116980766, + "pass@128": 0.9528678702668157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9186567164179105, + "pass@2": 0.9492878863556237, + "pass@4": 0.9682844965274856, + "pass@8": 0.9806815470310305, + "pass@16": 0.988738350593314, + "pass@32": 0.9937525351988347, + "pass@64": 0.9964109344316464, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9993303571428571, + "pass@2": 0.9999089566929135, + "pass@4": 0.9999980221222345, + "pass@8": 0.9999999993609641, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940324344023324, + "pass@2": 0.9959951992608066, + "pass@4": 0.9977383308136045, + "pass@8": 0.9990797355776098, + "pass@16": 0.9998081573699138, + "pass@32": 0.999992540612001, + "pass@64": 0.9999999988420875, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962993421052632, + "pass@2": 0.9970306959985263, + "pass@4": 0.9970758411229297, + "pass@8": 0.9970760233911381, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9735753676470589, + "pass@2": 0.9820366778601207, + "pass@4": 0.9875431995632898, + "pass@8": 0.9917505546806278, + "pass@16": 0.994889351721694, + "pass@32": 0.9974968435685705, + "pass@64": 0.9992203250542475, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9943069794952681, + "pass@2": 0.9978483072108101, + "pass@4": 0.9994458494344357, + "pass@8": 0.9999580528597197, + "pass@16": 0.9999998271433355, + "pass@32": 0.999999999999561, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9786392405063291, + "pass@2": 0.9860254130120601, + "pass@4": 0.9906770330528303, + "pass@8": 0.9942785219607105, + "pass@16": 0.9968841881317372, + "pass@32": 0.9988537819893496, + "pass@64": 0.9998879966359081, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9650431681681682, + "pass@2": 0.9763025821096688, + "pass@4": 0.982883417444441, + "pass@8": 0.9881286341578839, + "pass@16": 0.9924468973701125, + "pass@32": 0.9953769252358343, + "pass@64": 0.996775794068325, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9677451768488746, + "pass@2": 0.9829718079854168, + "pass@4": 0.9884183856037287, + "pass@8": 0.9913233575477782, + "pass@16": 0.9936550705400102, + "pass@32": 0.9960259491065468, + "pass@64": 0.998156322496061, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8957093253968254, + "pass@2": 0.9324045431821019, + "pass@4": 0.9530571291683771, + "pass@8": 0.9661005237823533, + "pass@16": 0.9753324158172761, + "pass@32": 0.9817093737531953, + "pass@64": 0.9852794285216058, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8548595183486238, + "pass@2": 0.9026862252883529, + "pass@4": 0.928716852767256, + "pass@8": 0.9458145843657703, + "pass@16": 0.9596806462958717, + "pass@32": 0.9727740288131387, + "pass@64": 0.9845103741794053, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.728515625, + "pass@2": 0.8026403926071737, + "pass@4": 0.854766033875395, + "pass@8": 0.8928687155866702, + "pass@16": 0.920794415945995, + "pass@32": 0.94152406634448, + "pass@64": 0.9565434779815419, + "pass@128": 0.9660493827160493 + }, + "14": { + "pass@1": 0.7207205118694362, + "pass@2": 0.7957049230122201, + "pass@4": 0.8462405605271146, + "pass@8": 0.8793643436005479, + "pass@16": 0.9014542133541928, + "pass@32": 0.9174294131525637, + "pass@64": 0.9303251689752986, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6839554398148148, + "pass@2": 0.7633861305531252, + "pass@4": 0.8194027031343298, + "pass@8": 0.8602466462398325, + "pass@16": 0.8887469421251796, + "pass@32": 0.9087115703307098, + "pass@64": 0.9239886621606587, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5756130162241888, + "pass@2": 0.6845900692170115, + "pass@4": 0.761188147112142, + "pass@8": 0.8149786242373147, + "pass@16": 0.8555630664100949, + "pass@32": 0.8888727828261448, + "pass@64": 0.9169712192641652, + "pass@128": 0.9351032448377581 + }, + "17": { + "pass@1": 0.4842808734939759, + "pass@2": 0.584871350559719, + "pass@4": 0.6645132360149255, + "pass@8": 0.7297835726231425, + "pass@16": 0.785105968775507, + "pass@32": 0.831487801164966, + "pass@64": 0.8694257138315328, + "pass@128": 0.9006024096385542 + }, + "18": { + "pass@1": 0.4754536290322581, + "pass@2": 0.5808114522479044, + "pass@4": 0.6585383135575791, + "pass@8": 0.7148880869953559, + "pass@16": 0.7581914940782433, + "pass@32": 0.7942661179834702, + "pass@64": 0.8252641405234724, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.325693919939577, + "pass@2": 0.4348405125246804, + "pass@4": 0.5347291311962133, + "pass@8": 0.6173469310391171, + "pass@16": 0.6816601314840767, + "pass@32": 0.7327839098583034, + "pass@64": 0.7748918502096696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3164179804804805, + "pass@2": 0.4194657255680876, + "pass@4": 0.5166342914905905, + "pass@8": 0.6003871057227763, + "pass@16": 0.6667422102319082, + "pass@32": 0.7176055052180353, + "pass@64": 0.7583024072323226, + "pass@128": 0.7867867867867868 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3a283d16c765bd40c29760607f863b6e2407b995 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62c594c47d85b2b293bc125617947e7269fceaa15a6f4bf739514400035e6cbc +size 1789722845 diff --git a/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json b/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..57e223e468ad0c108ab88295bed94a680a17a959 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9847204861111111, + "avg_response_len": 199.6684453125, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9847204861111111, + "pass@2": 0.9886822779964979, + "pass@4": 0.9911352969837098, + "pass@8": 0.9930673279925417, + "pass@16": 0.9946392213312069, + "pass@32": 0.9958122468158805, + "pass@64": 0.9965993654566087, + "pass@128": 0.9972222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9323125, + "pass@2": 0.9499406988188978, + "pass@4": 0.9621583696100489, + "pass@8": 0.9718907539598126, + "pass@16": 0.9791354444504873, + "pass@32": 0.9840437401738955, + "pass@64": 0.9871711591050699, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999828125, + "pass@2": 0.9999741633858267, + "pass@4": 0.999999438976378, + "pass@8": 0.9999999998576696, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998734375, + "pass@2": 0.999644438976378, + "pass@4": 0.9999579316647917, + "pass@8": 0.9999992906842913, + "pass@16": 0.9999999998783552, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995140625, + "pass@2": 0.9964927411417323, + "pass@4": 0.9973747537495314, + "pass@8": 0.9978259051966901, + "pass@16": 0.9979795297363963, + "pass@32": 0.9979997935880747, + "pass@64": 0.9979999999993951, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97978125, + "pass@2": 0.9833897637795275, + "pass@4": 0.9860962986501687, + "pass@8": 0.9884541169280153, + "pass@16": 0.990718014081456, + "pass@32": 0.9926000208673001, + "pass@64": 0.9939421204598907, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.995984375, + "pass@2": 0.9974320866141734, + "pass@4": 0.9982878877952756, + "pass@8": 0.9988890262019753, + "pass@16": 0.9993300424147991, + "pass@32": 0.9996835374287463, + "pass@64": 0.9999404387018788, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928359375, + "pass@2": 0.9942635334645668, + "pass@4": 0.9949447849643793, + "pass@8": 0.9955137921675251, + "pass@16": 0.9960397802495263, + "pass@32": 0.9964733721669579, + "pass@64": 0.9969998615760639, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.986671875, + "pass@2": 0.9904295029527562, + "pass@4": 0.9924173699850019, + "pass@8": 0.9938992253106746, + "pass@16": 0.9951051047857475, + "pass@32": 0.9960312897117236, + "pass@64": 0.9966908261256258, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9811953125, + "pass@2": 0.9865735728346459, + "pass@4": 0.9889808374578175, + "pass@8": 0.9911338416262176, + "pass@16": 0.9934450763840873, + "pass@32": 0.9954784674062254, + "pass@64": 0.9966498831415526, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9323125, + "2": 0.999828125, + "3": 0.998734375, + "4": 0.995140625, + "5": 0.97978125, + "6": 0.995984375, + "7": 0.9928359375, + "8": 0.986671875, + "9": 0.9811953125 + }, + "per_op_avg_response_len": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373242, + "answer_accuracy": 0.9847866008105369, + "avg_response_len": 190.43884561803443, + "resp_tokens_sum": 72177846, + "pass_at_k": { + "pass@1": 0.9847866008105369, + "pass@2": 0.9891751396766894, + "pass@4": 0.9916788649265851, + "pass@8": 0.9935028080912455, + "pass@16": 0.9950088418387877, + "pass@32": 0.9962314724797251, + "pass@64": 0.9970623347944672, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9362555309734514, + "pass@2": 0.9539477504471235, + "pass@4": 0.9651495407653686, + "pass@8": 0.973229583190713, + "pass@16": 0.9792803436377407, + "pass@32": 0.9841718673769317, + "pass@64": 0.9878246376873651, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983753822629969, + "pass@2": 0.9995236762262515, + "pass@4": 0.9999511242287376, + "pass@8": 0.9999994411862023, + "pass@16": 0.9999999999675167, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990673828125, + "pass@2": 0.9933247723917322, + "pass@4": 0.9952177153637045, + "pass@8": 0.9963542387601867, + "pass@16": 0.9968111376105945, + "pass@32": 0.9968743549630685, + "pass@64": 0.9968749999981098, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9707093253968254, + "pass@2": 0.9767396262967132, + "pass@4": 0.981683998131186, + "pass@8": 0.9859164493385404, + "pass@16": 0.9901710891964574, + "pass@32": 0.9936881458577858, + "pass@64": 0.995845992043236, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9987512671507378, + "pass@4": 0.9995554992377451, + "pass@8": 0.999939295718634, + "pass@16": 0.9999991353050729, + "pass@32": 0.9999999999556102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958394970414202, + "pass@2": 0.9969697124120579, + "pass@4": 0.9970408646219519, + "pass@8": 0.9970414201059147, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9894080528846154, + "pass@2": 0.9919966434484152, + "pass@4": 0.9931632229865497, + "pass@8": 0.9938714043202131, + "pass@16": 0.9943410041176739, + "pass@32": 0.9949967186169365, + "pass@64": 0.9959998990510801, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9859595070422535, + "pass@2": 0.9918774952866809, + "pass@4": 0.9937267724456977, + "pass@8": 0.9954155716338281, + "pass@16": 0.9975424861780133, + "pass@32": 0.9993143415342073, + "pass@64": 0.9999593384604206, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 384296, + "answer_accuracy": 0.9837196920052425, + "avg_response_len": 197.6666734927916, + "resp_tokens_sum": 77219672, + "pass_at_k": { + "pass@1": 0.9837196920052425, + "pass@2": 0.9873512979742215, + "pass@4": 0.9898586991214896, + "pass@8": 0.9920136735719006, + "pass@16": 0.9938161565342344, + "pass@32": 0.9951304248579761, + "pass@64": 0.9960841018262011, + "pass@128": 0.997051114023591 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9209403757668712, + "pass@2": 0.9380713794019612, + "pass@4": 0.9512705088436028, + "pass@8": 0.9631955401424825, + "pass@16": 0.9726077613588666, + "pass@32": 0.9782359909847477, + "pass@64": 0.9813867257620852, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9994191422094966, + "pass@4": 0.9999210212359819, + "pass@8": 0.9999984042922265, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9975268121301775, + "pass@2": 0.9989032724921958, + "pass@4": 0.9997191858968812, + "pass@8": 0.9999779550101494, + "pass@16": 0.999999898523686, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875679347826087, + "pass@2": 0.9887766746547987, + "pass@4": 0.9894790216440336, + "pass@8": 0.9903547461693829, + "pass@16": 0.9916253749261505, + "pass@32": 0.9931572503524777, + "pass@64": 0.9949313065753761, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9943364971346705, + "pass@2": 0.9965329089863052, + "pass@4": 0.9980393601408706, + "pass@8": 0.9990811493323517, + "pass@16": 0.9997493490657728, + "pass@32": 0.9999854792814306, + "pass@64": 0.999999993573057, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934293713872833, + "pass@2": 0.9942072077966412, + "pass@4": 0.9943974088679668, + "pass@8": 0.9945809233687215, + "pass@16": 0.994942196531792, + "pass@32": 0.9956647398843931, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9836707746478873, + "pass@2": 0.987972371631363, + "pass@4": 0.9905331931219864, + "pass@8": 0.9926688812257355, + "pass@16": 0.9945746668596219, + "pass@32": 0.9961092494376016, + "pass@64": 0.9970112564626833, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9756034805389222, + "pass@2": 0.9805492361733227, + "pass@4": 0.9840381910532142, + "pass@8": 0.9872988101167206, + "pass@16": 0.9901482935355289, + "pass@32": 0.992455558127094, + "pass@64": 0.9937913521118914, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 376860, + "answer_accuracy": 0.9856775192500837, + "avg_response_len": 210.86303931620355, + "resp_tokens_sum": 80620531, + "pass_at_k": { + "pass@1": 0.9856775192500837, + "pass@2": 0.9895536498053247, + "pass@4": 0.9919008383284197, + "pass@8": 0.9937122214373127, + "pass@16": 0.9951137936236103, + "pass@32": 0.9960933293150033, + "pass@64": 0.9966669022462312, + "pass@128": 0.9969869434214932 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9393889925373134, + "pass@2": 0.9574362439769657, + "pass@4": 0.9697268340524599, + "pass@8": 0.9789975497663034, + "pass@16": 0.9853411276247855, + "pass@32": 0.9895658032002611, + "pass@64": 0.9921389084495922, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99953125, + "pass@2": 0.9999261811023622, + "pass@4": 0.9999983970753656, + "pass@8": 0.9999999995933421, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999763263010492, + "pass@4": 0.9999999330440837, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969618055555556, + "pass@2": 0.9970745844269466, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9802849264705882, + "pass@2": 0.984084790412228, + "pass@4": 0.9867516963872165, + "pass@8": 0.9888766116409934, + "pass@16": 0.9903040195148527, + "pass@32": 0.9910264810308619, + "pass@64": 0.9911745064640191, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9970321077274646, + "pass@4": 0.9972259128097947, + "pass@8": 0.9975709158200651, + "pass@16": 0.9981634334351739, + "pass@32": 0.9990176819380228, + "pass@64": 0.9998121165453686, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.988973496835443, + "pass@2": 0.9914306226951061, + "pass@4": 0.993302117891909, + "pass@8": 0.9949012426776844, + "pass@16": 0.9961701906630579, + "pass@32": 0.9967511777435377, + "pass@64": 0.996835004987544, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9873076201201201, + "pass@2": 0.9915806554586083, + "pass@4": 0.9937272097744536, + "pass@8": 0.9952369199027994, + "pass@16": 0.996386500798402, + "pass@32": 0.9969175073660387, + "pass@64": 0.9969965813136217, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9869891225915893, + "pass@4": 0.9888716316007123, + "pass@8": 0.9903649875151894, + "pass@16": 0.9923085651126249, + "pass@32": 0.9943463657464062, + "pass@64": 0.9959421427097476, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.40368125, + "avg_response_len": 320.694090625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.40368125, + "pass@2": 0.46667384350394003, + "pass@4": 0.5202853888451408, + "pass@8": 0.566301432358076, + "pass@16": 0.60583017925103, + "pass@32": 0.6399910807570778, + "pass@64": 0.6699377993651461, + "pass@128": 0.6963 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.894578125, + "pass@2": 0.9257929379921268, + "pass@4": 0.9442590304649411, + "pass@8": 0.9561462437618224, + "pass@16": 0.9641911885882701, + "pass@32": 0.9700611686252756, + "pass@64": 0.9752575710913312, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.812484375, + "pass@2": 0.8672572588582684, + "pass@4": 0.8985465786464193, + "pass@8": 0.9196361714553097, + "pass@16": 0.9348869636510788, + "pass@32": 0.94673284148625, + "pass@64": 0.9576580920053203, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.5978046875, + "pass@2": 0.6833599901574804, + "pass@4": 0.7455164355080617, + "pass@8": 0.7936291630299753, + "pass@16": 0.8329134533248633, + "pass@32": 0.8646922500467323, + "pass@64": 0.8900894458435458, + "pass@128": 0.911 + }, + "14": { + "pass@1": 0.45890625, + "pass@2": 0.5430552411417325, + "pass@4": 0.6134239566929134, + "pass@8": 0.6705658372251655, + "pass@16": 0.7163013175856843, + "pass@32": 0.7538964282786471, + "pass@64": 0.7852020700568273, + "pass@128": 0.813 + }, + "15": { + "pass@1": 0.3480078125, + "pass@2": 0.4288660187007878, + "pass@4": 0.49983243916385395, + "pass@8": 0.561707925390508, + "pass@16": 0.6160022823168169, + "pass@32": 0.6632416916320305, + "pass@64": 0.7017246400131141, + "pass@128": 0.73 + }, + "16": { + "pass@1": 0.237484375, + "pass@2": 0.3018716781496062, + "pass@4": 0.36146487692163465, + "pass@8": 0.41654215784915727, + "pass@16": 0.467808763276434, + "pass@32": 0.5155089371433209, + "pass@64": 0.5604432358720062, + "pass@128": 0.602 + }, + "17": { + "pass@1": 0.2055234375, + "pass@2": 0.26673203740157486, + "pass@4": 0.3232418611736032, + "pass@8": 0.37323857044448777, + "pass@16": 0.41837508369464593, + "pass@32": 0.4601245400414008, + "pass@64": 0.4978334061713972, + "pass@128": 0.53 + }, + "18": { + "pass@1": 0.1816796875, + "pass@2": 0.24128063484251946, + "pass@4": 0.30061707564679396, + "pass@8": 0.35639546958098467, + "pass@16": 0.40555332405555866, + "pass@32": 0.44808731643199917, + "pass@64": 0.4868599251210825, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.150984375, + "pass@2": 0.2036835629921259, + "pass@4": 0.25508346662917125, + "pass@8": 0.30095403253935643, + "pass@16": 0.3403886075532727, + "pass@32": 0.37524624378973637, + "pass@64": 0.40789729833590765, + "pass@128": 0.44 + }, + "20": { + "pass@1": 0.149359375, + "pass@2": 0.20483907480314964, + "pass@4": 0.2608681676040494, + "pass@8": 0.31419875230396843, + "pass@16": 0.3618808084635515, + "pass@32": 0.4023193900953739, + "pass@64": 0.436412309140885, + "pass@128": 0.464 + } + }, + "per_op_accuracy": { + "11": 0.894578125, + "12": 0.812484375, + "13": 0.5978046875, + "14": 0.45890625, + "15": 0.3480078125, + "16": 0.237484375, + "17": 0.2055234375, + "18": 0.1816796875, + "19": 0.150984375, + "20": 0.149359375 + }, + "per_op_avg_response_len": { + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 174941, + "answer_accuracy": 0.40615945393759284, + "avg_response_len": 304.98383172362554, + "resp_tokens_sum": 131362636, + "pass_at_k": { + "pass@1": 0.40615945393759284, + "pass@2": 0.4685217793169621, + "pass@4": 0.5206486526201332, + "pass@8": 0.564898064058231, + "pass@16": 0.6029430039326625, + "pass@32": 0.6357447987596025, + "pass@64": 0.6650622830418412, + "pass@128": 0.6921248142644874 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8889872521246459, + "pass@2": 0.9209322232383845, + "pass@4": 0.9403707372980644, + "pass@8": 0.9523073338351158, + "pass@16": 0.9599911546995261, + "pass@32": 0.9656664426486973, + "pass@64": 0.97105654419183, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.807468220338983, + "pass@2": 0.8675287907602651, + "pass@4": 0.9025985565363646, + "pass@8": 0.9234312139724606, + "pass@16": 0.9362103618765187, + "pass@32": 0.9450079869218969, + "pass@64": 0.9525377320034726, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.6421923981191222, + "pass@2": 0.7228887178189717, + "pass@4": 0.7779022200986624, + "pass@8": 0.8182676410254136, + "pass@16": 0.8505300132234627, + "pass@32": 0.8773909089095185, + "pass@64": 0.9017227414682624, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.4485007788161994, + "pass@2": 0.5347201780852163, + "pass@4": 0.6061306235669136, + "pass@8": 0.6653760404004355, + "pass@16": 0.7137935405594372, + "pass@32": 0.7534356762985341, + "pass@64": 0.7870531477671298, + "pass@128": 0.8130841121495327 + }, + "15": { + "pass@1": 0.3333408598265896, + "pass@2": 0.41465276671521556, + "pass@4": 0.4854122544653017, + "pass@8": 0.5467994374688726, + "pass@16": 0.6020372401229079, + "pass@32": 0.6496721898260739, + "pass@64": 0.686791777517624, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2532258064516129, + "pass@2": 0.3180189547879094, + "pass@4": 0.374151785260713, + "pass@8": 0.42593478016771535, + "pass@16": 0.4749139463216263, + "pass@32": 0.5228389769740335, + "pass@64": 0.5721693101663297, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19495122832369943, + "pass@2": 0.2523230195484954, + "pass@4": 0.3023951223040761, + "pass@8": 0.3470740122741218, + "pass@16": 0.39059013486586935, + "pass@32": 0.4324684907446193, + "pass@64": 0.46828111323798854, + "pass@128": 0.49421965317919075 + }, + "18": { + "pass@1": 0.16796875, + "pass@2": 0.22441437007874018, + "pass@4": 0.28228091943864153, + "pass@8": 0.33871351937233285, + "pass@16": 0.38905522009524307, + "pass@32": 0.4321079296175608, + "pass@64": 0.47367519903850336, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1451998873873874, + "pass@2": 0.20028788631150843, + "pass@4": 0.2549552702308609, + "pass@8": 0.30319246623896356, + "pass@16": 0.34293555954657895, + "pass@32": 0.3751435020629878, + "pass@64": 0.4044465014859663, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.1495870870870871, + "pass@2": 0.20280615260930232, + "pass@4": 0.25683599347378866, + "pass@8": 0.30743826429217297, + "pass@16": 0.3517171120545327, + "pass@32": 0.38878767049319296, + "pass@64": 0.4209157334351972, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 170169, + "answer_accuracy": 0.39531528768956287, + "avg_response_len": 315.663583946625, + "resp_tokens_sum": 135881809, + "pass_at_k": { + "pass@1": 0.39531528768956287, + "pass@2": 0.4581659987918574, + "pass@4": 0.5125038153404343, + "pass@8": 0.5601027407054775, + "pass@16": 0.6017583420304699, + "pass@32": 0.638086461585684, + "pass@64": 0.6691368394613612, + "pass@128": 0.6946179006839132 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8997552710843374, + "pass@2": 0.9309293028412866, + "pass@4": 0.9496467379642303, + "pass@8": 0.9619478037609, + "pass@16": 0.9701481675058946, + "pass@32": 0.9759861111837409, + "pass@64": 0.981159773824385, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8052752742946708, + "pass@2": 0.8523317361093967, + "pass@4": 0.8786195395246442, + "pass@8": 0.8979108712169228, + "pass@16": 0.9138855036221462, + "pass@32": 0.9282479851031312, + "pass@64": 0.9436447705145927, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5835084033613446, + "pass@2": 0.6718991238227576, + "pass@4": 0.7351696928115078, + "pass@8": 0.7855705437580075, + "pass@16": 0.8282254349495101, + "pass@32": 0.8632778064510783, + "pass@64": 0.8906273038627206, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.4359466374269006, + "pass@2": 0.5169355372519222, + "pass@4": 0.5883138300146694, + "pass@8": 0.6484225858141512, + "pass@16": 0.6976308033859067, + "pass@32": 0.7384376268134236, + "pass@64": 0.7720516569697223, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.3393465909090909, + "pass@2": 0.4214902469577665, + "pass@4": 0.49189631068843637, + "pass@8": 0.5528083984205381, + "pass@16": 0.6063339208685132, + "pass@32": 0.6533444169232615, + "pass@64": 0.6909033146987849, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.22008547008547008, + "pass@2": 0.2828445022096597, + "pass@4": 0.34362964138029767, + "pass@8": 0.4025933786277824, + "pass@16": 0.45909006271494923, + "pass@32": 0.509012724160067, + "pass@64": 0.5514726875161375, + "pass@128": 0.5897435897435898 + }, + "17": { + "pass@1": 0.20479910714285715, + "pass@2": 0.26985353902772996, + "pass@4": 0.3339852589272611, + "pass@8": 0.39210840306068734, + "pass@16": 0.44226737100715613, + "pass@32": 0.4864598585876851, + "pass@64": 0.5269995343426999, + "pass@128": 0.562111801242236 + }, + "18": { + "pass@1": 0.18855698529411766, + "pass@2": 0.24998842056507656, + "pass@4": 0.30994159333024557, + "pass@8": 0.36606403192727205, + "pass@16": 0.4167227598892368, + "pass@32": 0.4612764576013937, + "pass@64": 0.4994967680463235, + "pass@128": 0.5294117647058824 + }, + "19": { + "pass@1": 0.1353701636904762, + "pass@2": 0.1830628105080615, + "pass@4": 0.23144565969432382, + "pass@8": 0.2755576222539975, + "pass@16": 0.3158109715954468, + "pass@32": 0.35526680353587153, + "pass@64": 0.39177456043247216, + "pass@128": 0.4226190476190476 + }, + "20": { + "pass@1": 0.15468282185628743, + "pass@2": 0.21383758074402373, + "pass@4": 0.2715573271342579, + "pass@8": 0.32506282832444283, + "pass@16": 0.37242304719707847, + "pass@32": 0.41287302118538943, + "pass@64": 0.4455683819577822, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 171602, + "answer_accuracy": 0.4097312423594132, + "avg_response_len": 342.0212957480135, + "resp_tokens_sum": 143243991, + "pass_at_k": { + "pass@1": 0.4097312423594132, + "pass@2": 0.47351784648557116, + "pass@4": 0.5279097926023316, + "pass@8": 0.574115776599107, + "pass@16": 0.612984498786154, + "pass@32": 0.6463156446919366, + "pass@64": 0.6757751283945757, + "pass@128": 0.7023227383863081 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8953869047619047, + "pass@2": 0.9258264591926012, + "pass@4": 0.9429379149034938, + "pass@8": 0.9543336002203412, + "pass@16": 0.9626194265631142, + "pass@32": 0.9687413506580436, + "pass@64": 0.9737446542283149, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8249474388379205, + "pass@2": 0.8815236792361963, + "pass@4": 0.9135995612360381, + "pass@8": 0.9367215712258752, + "pass@16": 0.9539419568542121, + "pass@32": 0.9666327427155964, + "pass@64": 0.9768717219631041, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5698543595679012, + "pass@2": 0.6570694505200732, + "pass@4": 0.7250310091446903, + "pass@8": 0.7782503130285764, + "pass@16": 0.8207342556469278, + "pass@32": 0.8537480654369178, + "pass@64": 0.878043036531414, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.4921179525222552, + "pass@2": 0.5775018107899714, + "pass@4": 0.64585369929649, + "pass@8": 0.69798102049907, + "pass@16": 0.7376375558104614, + "pass@32": 0.770023465331237, + "pass@64": 0.7967844004152322, + "pass@128": 0.8219584569732937 + }, + "15": { + "pass@1": 0.3724922839506173, + "pass@2": 0.4515567281520361, + "pass@4": 0.5233148660236916, + "pass@8": 0.5866930510725332, + "pass@16": 0.6407629732953167, + "pass@32": 0.6878131369368291, + "pass@64": 0.7286931825352366, + "pass@128": 0.7623456790123457 + }, + "16": { + "pass@1": 0.24110435103244837, + "pass@2": 0.3068064362529904, + "pass@4": 0.3683298506381394, + "pass@8": 0.42239557551272405, + "pass@16": 0.4703387253798914, + "pass@32": 0.5155321182925883, + "pass@64": 0.5590083669683755, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.21724397590361447, + "pass@2": 0.2787211839483919, + "pass@4": 0.33454787795727337, + "pass@8": 0.38220498919307505, + "pass@16": 0.4241590468878052, + "pass@32": 0.4634047222244833, + "pass@64": 0.5003442497973006, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18961693548387096, + "pass@2": 0.2507727171704342, + "pass@4": 0.3110922971322133, + "pass@8": 0.3657547320819236, + "pass@16": 0.41192986664478526, + "pass@32": 0.45166304993993356, + "pass@64": 0.4878861429737296, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17265388972809667, + "pass@2": 0.22803200157004544, + "pass@4": 0.2792073111329362, + "pass@8": 0.32448211542126465, + "pass@16": 0.36277516546281496, + "pass@32": 0.39563085079972393, + "pass@64": 0.4277352297153177, + "pass@128": 0.4622356495468278 + }, + "20": { + "pass@1": 0.14379222972972974, + "pass@2": 0.1978464685157598, + "pass@4": 0.25417908262593303, + "pass@8": 0.310062539442377, + "pass@16": 0.3614706077644692, + "pass@32": 0.40526578602168234, + "pass@64": 0.44272531631551015, + "pass@128": 0.47147147147147145 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6789103618421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6789103618421053, + "pass@2": 0.7139409966846074, + "pass@4": 0.7433195558581301, + "pass@8": 0.7684536987112555, + "pass@16": 0.790002883394277, + "pass@32": 0.8085379488902074, + "pass@64": 0.8246722254084696, + "pass@128": 0.8388421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9323125, + "pass@2": 0.9499406988188978, + "pass@4": 0.9621583696100489, + "pass@8": 0.9718907539598126, + "pass@16": 0.9791354444504873, + "pass@32": 0.9840437401738955, + "pass@64": 0.9871711591050699, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999828125, + "pass@2": 0.9999741633858267, + "pass@4": 0.999999438976378, + "pass@8": 0.9999999998576696, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998734375, + "pass@2": 0.999644438976378, + "pass@4": 0.9999579316647917, + "pass@8": 0.9999992906842913, + "pass@16": 0.9999999998783552, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995140625, + "pass@2": 0.9964927411417323, + "pass@4": 0.9973747537495314, + "pass@8": 0.9978259051966901, + "pass@16": 0.9979795297363963, + "pass@32": 0.9979997935880747, + "pass@64": 0.9979999999993951, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97978125, + "pass@2": 0.9833897637795275, + "pass@4": 0.9860962986501687, + "pass@8": 0.9884541169280153, + "pass@16": 0.990718014081456, + "pass@32": 0.9926000208673001, + "pass@64": 0.9939421204598907, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.995984375, + "pass@2": 0.9974320866141734, + "pass@4": 0.9982878877952756, + "pass@8": 0.9988890262019753, + "pass@16": 0.9993300424147991, + "pass@32": 0.9996835374287463, + "pass@64": 0.9999404387018788, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928359375, + "pass@2": 0.9942635334645668, + "pass@4": 0.9949447849643793, + "pass@8": 0.9955137921675251, + "pass@16": 0.9960397802495263, + "pass@32": 0.9964733721669579, + "pass@64": 0.9969998615760639, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.986671875, + "pass@2": 0.9904295029527562, + "pass@4": 0.9924173699850019, + "pass@8": 0.9938992253106746, + "pass@16": 0.9951051047857475, + "pass@32": 0.9960312897117236, + "pass@64": 0.9966908261256258, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9811953125, + "pass@2": 0.9865735728346459, + "pass@4": 0.9889808374578175, + "pass@8": 0.9911338416262176, + "pass@16": 0.9934450763840873, + "pass@32": 0.9954784674062254, + "pass@64": 0.9966498831415526, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.894578125, + "pass@2": 0.9257929379921268, + "pass@4": 0.9442590304649411, + "pass@8": 0.9561462437618224, + "pass@16": 0.9641911885882701, + "pass@32": 0.9700611686252756, + "pass@64": 0.9752575710913312, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.812484375, + "pass@2": 0.8672572588582684, + "pass@4": 0.8985465786464193, + "pass@8": 0.9196361714553097, + "pass@16": 0.9348869636510788, + "pass@32": 0.94673284148625, + "pass@64": 0.9576580920053203, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.5978046875, + "pass@2": 0.6833599901574804, + "pass@4": 0.7455164355080617, + "pass@8": 0.7936291630299753, + "pass@16": 0.8329134533248633, + "pass@32": 0.8646922500467323, + "pass@64": 0.8900894458435458, + "pass@128": 0.911 + }, + "14": { + "pass@1": 0.45890625, + "pass@2": 0.5430552411417325, + "pass@4": 0.6134239566929134, + "pass@8": 0.6705658372251655, + "pass@16": 0.7163013175856843, + "pass@32": 0.7538964282786471, + "pass@64": 0.7852020700568273, + "pass@128": 0.813 + }, + "15": { + "pass@1": 0.3480078125, + "pass@2": 0.4288660187007878, + "pass@4": 0.49983243916385395, + "pass@8": 0.561707925390508, + "pass@16": 0.6160022823168169, + "pass@32": 0.6632416916320305, + "pass@64": 0.7017246400131141, + "pass@128": 0.73 + }, + "16": { + "pass@1": 0.237484375, + "pass@2": 0.3018716781496062, + "pass@4": 0.36146487692163465, + "pass@8": 0.41654215784915727, + "pass@16": 0.467808763276434, + "pass@32": 0.5155089371433209, + "pass@64": 0.5604432358720062, + "pass@128": 0.602 + }, + "17": { + "pass@1": 0.2055234375, + "pass@2": 0.26673203740157486, + "pass@4": 0.3232418611736032, + "pass@8": 0.37323857044448777, + "pass@16": 0.41837508369464593, + "pass@32": 0.4601245400414008, + "pass@64": 0.4978334061713972, + "pass@128": 0.53 + }, + "18": { + "pass@1": 0.1816796875, + "pass@2": 0.24128063484251946, + "pass@4": 0.30061707564679396, + "pass@8": 0.35639546958098467, + "pass@16": 0.40555332405555866, + "pass@32": 0.44808731643199917, + "pass@64": 0.4868599251210825, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.150984375, + "pass@2": 0.2036835629921259, + "pass@4": 0.25508346662917125, + "pass@8": 0.30095403253935643, + "pass@16": 0.3403886075532727, + "pass@32": 0.37524624378973637, + "pass@64": 0.40789729833590765, + "pass@128": 0.44 + }, + "20": { + "pass@1": 0.149359375, + "pass@2": 0.20483907480314964, + "pass@4": 0.2608681676040494, + "pass@8": 0.31419875230396843, + "pass@16": 0.3618808084635515, + "pass@32": 0.4023193900953739, + "pass@64": 0.436412309140885, + "pass@128": 0.464 + } + }, + "per_op_accuracy": { + "10": 0.9323125, + "2": 0.999828125, + "3": 0.998734375, + "4": 0.995140625, + "5": 0.97978125, + "6": 0.995984375, + "7": 0.9928359375, + "8": 0.986671875, + "9": 0.9811953125, + "11": 0.894578125, + "12": 0.812484375, + "13": 0.5978046875, + "14": 0.45890625, + "15": 0.3480078125, + "16": 0.237484375, + "17": 0.2055234375, + "18": 0.1816796875, + "19": 0.150984375, + "20": 0.149359375 + }, + "per_op_avg_response_len": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625, + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625, + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 548183, + "answer_accuracy": 0.6769964728896617, + "avg_response_len": 251.36895599509958, + "resp_tokens_sum": 203540482, + "pass_at_k": { + "pass@1": 0.6769964728896617, + "pass@2": 0.7122231071742398, + "pass@4": 0.7411229584436191, + "pass@8": 0.7655143535115593, + "pass@16": 0.7864565900913838, + "pass@32": 0.8044771795508289, + "pass@64": 0.8204609794123039, + "pass@128": 0.8351248814416693 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9362555309734514, + "pass@2": 0.9539477504471235, + "pass@4": 0.9651495407653686, + "pass@8": 0.973229583190713, + "pass@16": 0.9792803436377407, + "pass@32": 0.9841718673769317, + "pass@64": 0.9878246376873651, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983753822629969, + "pass@2": 0.9995236762262515, + "pass@4": 0.9999511242287376, + "pass@8": 0.9999994411862023, + "pass@16": 0.9999999999675167, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990673828125, + "pass@2": 0.9933247723917322, + "pass@4": 0.9952177153637045, + "pass@8": 0.9963542387601867, + "pass@16": 0.9968111376105945, + "pass@32": 0.9968743549630685, + "pass@64": 0.9968749999981098, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9707093253968254, + "pass@2": 0.9767396262967132, + "pass@4": 0.981683998131186, + "pass@8": 0.9859164493385404, + "pass@16": 0.9901710891964574, + "pass@32": 0.9936881458577858, + "pass@64": 0.995845992043236, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9987512671507378, + "pass@4": 0.9995554992377451, + "pass@8": 0.999939295718634, + "pass@16": 0.9999991353050729, + "pass@32": 0.9999999999556102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958394970414202, + "pass@2": 0.9969697124120579, + "pass@4": 0.9970408646219519, + "pass@8": 0.9970414201059147, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9894080528846154, + "pass@2": 0.9919966434484152, + "pass@4": 0.9931632229865497, + "pass@8": 0.9938714043202131, + "pass@16": 0.9943410041176739, + "pass@32": 0.9949967186169365, + "pass@64": 0.9959998990510801, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9859595070422535, + "pass@2": 0.9918774952866809, + "pass@4": 0.9937267724456977, + "pass@8": 0.9954155716338281, + "pass@16": 0.9975424861780133, + "pass@32": 0.9993143415342073, + "pass@64": 0.9999593384604206, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8889872521246459, + "pass@2": 0.9209322232383845, + "pass@4": 0.9403707372980644, + "pass@8": 0.9523073338351158, + "pass@16": 0.9599911546995261, + "pass@32": 0.9656664426486973, + "pass@64": 0.97105654419183, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.807468220338983, + "pass@2": 0.8675287907602651, + "pass@4": 0.9025985565363646, + "pass@8": 0.9234312139724606, + "pass@16": 0.9362103618765187, + "pass@32": 0.9450079869218969, + "pass@64": 0.9525377320034726, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.6421923981191222, + "pass@2": 0.7228887178189717, + "pass@4": 0.7779022200986624, + "pass@8": 0.8182676410254136, + "pass@16": 0.8505300132234627, + "pass@32": 0.8773909089095185, + "pass@64": 0.9017227414682624, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.4485007788161994, + "pass@2": 0.5347201780852163, + "pass@4": 0.6061306235669136, + "pass@8": 0.6653760404004355, + "pass@16": 0.7137935405594372, + "pass@32": 0.7534356762985341, + "pass@64": 0.7870531477671298, + "pass@128": 0.8130841121495327 + }, + "15": { + "pass@1": 0.3333408598265896, + "pass@2": 0.41465276671521556, + "pass@4": 0.4854122544653017, + "pass@8": 0.5467994374688726, + "pass@16": 0.6020372401229079, + "pass@32": 0.6496721898260739, + "pass@64": 0.686791777517624, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2532258064516129, + "pass@2": 0.3180189547879094, + "pass@4": 0.374151785260713, + "pass@8": 0.42593478016771535, + "pass@16": 0.4749139463216263, + "pass@32": 0.5228389769740335, + "pass@64": 0.5721693101663297, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19495122832369943, + "pass@2": 0.2523230195484954, + "pass@4": 0.3023951223040761, + "pass@8": 0.3470740122741218, + "pass@16": 0.39059013486586935, + "pass@32": 0.4324684907446193, + "pass@64": 0.46828111323798854, + "pass@128": 0.49421965317919075 + }, + "18": { + "pass@1": 0.16796875, + "pass@2": 0.22441437007874018, + "pass@4": 0.28228091943864153, + "pass@8": 0.33871351937233285, + "pass@16": 0.38905522009524307, + "pass@32": 0.4321079296175608, + "pass@64": 0.47367519903850336, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1451998873873874, + "pass@2": 0.20028788631150843, + "pass@4": 0.2549552702308609, + "pass@8": 0.30319246623896356, + "pass@16": 0.34293555954657895, + "pass@32": 0.3751435020629878, + "pass@64": 0.4044465014859663, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.1495870870870871, + "pass@2": 0.20280615260930232, + "pass@4": 0.25683599347378866, + "pass@8": 0.30743826429217297, + "pass@16": 0.3517171120545327, + "pass@32": 0.38878767049319296, + "pass@64": 0.4209157334351972, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 554465, + "answer_accuracy": 0.6752545303975058, + "avg_response_len": 259.52538118667184, + "resp_tokens_sum": 213101481, + "pass_at_k": { + "pass@1": 0.6752545303975058, + "pass@2": 0.7099311637341081, + "pass@4": 0.7396101450831859, + "pass@8": 0.765588659194692, + "pass@16": 0.788283743412472, + "pass@32": 0.8079536752890428, + "pass@64": 0.8246852486176389, + "pass@128": 0.8385035074045206 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9209403757668712, + "pass@2": 0.9380713794019612, + "pass@4": 0.9512705088436028, + "pass@8": 0.9631955401424825, + "pass@16": 0.9726077613588666, + "pass@32": 0.9782359909847477, + "pass@64": 0.9813867257620852, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9994191422094966, + "pass@4": 0.9999210212359819, + "pass@8": 0.9999984042922265, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9975268121301775, + "pass@2": 0.9989032724921958, + "pass@4": 0.9997191858968812, + "pass@8": 0.9999779550101494, + "pass@16": 0.999999898523686, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875679347826087, + "pass@2": 0.9887766746547987, + "pass@4": 0.9894790216440336, + "pass@8": 0.9903547461693829, + "pass@16": 0.9916253749261505, + "pass@32": 0.9931572503524777, + "pass@64": 0.9949313065753761, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9943364971346705, + "pass@2": 0.9965329089863052, + "pass@4": 0.9980393601408706, + "pass@8": 0.9990811493323517, + "pass@16": 0.9997493490657728, + "pass@32": 0.9999854792814306, + "pass@64": 0.999999993573057, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934293713872833, + "pass@2": 0.9942072077966412, + "pass@4": 0.9943974088679668, + "pass@8": 0.9945809233687215, + "pass@16": 0.994942196531792, + "pass@32": 0.9956647398843931, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9836707746478873, + "pass@2": 0.987972371631363, + "pass@4": 0.9905331931219864, + "pass@8": 0.9926688812257355, + "pass@16": 0.9945746668596219, + "pass@32": 0.9961092494376016, + "pass@64": 0.9970112564626833, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9756034805389222, + "pass@2": 0.9805492361733227, + "pass@4": 0.9840381910532142, + "pass@8": 0.9872988101167206, + "pass@16": 0.9901482935355289, + "pass@32": 0.992455558127094, + "pass@64": 0.9937913521118914, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8997552710843374, + "pass@2": 0.9309293028412866, + "pass@4": 0.9496467379642303, + "pass@8": 0.9619478037609, + "pass@16": 0.9701481675058946, + "pass@32": 0.9759861111837409, + "pass@64": 0.981159773824385, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8052752742946708, + "pass@2": 0.8523317361093967, + "pass@4": 0.8786195395246442, + "pass@8": 0.8979108712169228, + "pass@16": 0.9138855036221462, + "pass@32": 0.9282479851031312, + "pass@64": 0.9436447705145927, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5835084033613446, + "pass@2": 0.6718991238227576, + "pass@4": 0.7351696928115078, + "pass@8": 0.7855705437580075, + "pass@16": 0.8282254349495101, + "pass@32": 0.8632778064510783, + "pass@64": 0.8906273038627206, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.4359466374269006, + "pass@2": 0.5169355372519222, + "pass@4": 0.5883138300146694, + "pass@8": 0.6484225858141512, + "pass@16": 0.6976308033859067, + "pass@32": 0.7384376268134236, + "pass@64": 0.7720516569697223, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.3393465909090909, + "pass@2": 0.4214902469577665, + "pass@4": 0.49189631068843637, + "pass@8": 0.5528083984205381, + "pass@16": 0.6063339208685132, + "pass@32": 0.6533444169232615, + "pass@64": 0.6909033146987849, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.22008547008547008, + "pass@2": 0.2828445022096597, + "pass@4": 0.34362964138029767, + "pass@8": 0.4025933786277824, + "pass@16": 0.45909006271494923, + "pass@32": 0.509012724160067, + "pass@64": 0.5514726875161375, + "pass@128": 0.5897435897435898 + }, + "17": { + "pass@1": 0.20479910714285715, + "pass@2": 0.26985353902772996, + "pass@4": 0.3339852589272611, + "pass@8": 0.39210840306068734, + "pass@16": 0.44226737100715613, + "pass@32": 0.4864598585876851, + "pass@64": 0.5269995343426999, + "pass@128": 0.562111801242236 + }, + "18": { + "pass@1": 0.18855698529411766, + "pass@2": 0.24998842056507656, + "pass@4": 0.30994159333024557, + "pass@8": 0.36606403192727205, + "pass@16": 0.4167227598892368, + "pass@32": 0.4612764576013937, + "pass@64": 0.4994967680463235, + "pass@128": 0.5294117647058824 + }, + "19": { + "pass@1": 0.1353701636904762, + "pass@2": 0.1830628105080615, + "pass@4": 0.23144565969432382, + "pass@8": 0.2755576222539975, + "pass@16": 0.3158109715954468, + "pass@32": 0.35526680353587153, + "pass@64": 0.39177456043247216, + "pass@128": 0.4226190476190476 + }, + "20": { + "pass@1": 0.15468282185628743, + "pass@2": 0.21383758074402373, + "pass@4": 0.2715573271342579, + "pass@8": 0.32506282832444283, + "pass@16": 0.37242304719707847, + "pass@32": 0.41287302118538943, + "pass@64": 0.4455683819577822, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 548462, + "answer_accuracy": 0.6845916879693241, + "avg_response_len": 279.4282757828727, + "resp_tokens_sum": 223864522, + "pass_at_k": { + "pass@1": 0.6845916879693241, + "pass@2": 0.7197870499551496, + "pass@4": 0.7493415314717663, + "pass@8": 0.7743609564571847, + "pass@16": 0.7953491263112427, + "pass@32": 0.8132410231819698, + "pass@64": 0.8289152032459771, + "pass@128": 0.8429461575331523 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9393889925373134, + "pass@2": 0.9574362439769657, + "pass@4": 0.9697268340524599, + "pass@8": 0.9789975497663034, + "pass@16": 0.9853411276247855, + "pass@32": 0.9895658032002611, + "pass@64": 0.9921389084495922, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99953125, + "pass@2": 0.9999261811023622, + "pass@4": 0.9999983970753656, + "pass@8": 0.9999999995933421, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999763263010492, + "pass@4": 0.9999999330440837, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969618055555556, + "pass@2": 0.9970745844269466, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9802849264705882, + "pass@2": 0.984084790412228, + "pass@4": 0.9867516963872165, + "pass@8": 0.9888766116409934, + "pass@16": 0.9903040195148527, + "pass@32": 0.9910264810308619, + "pass@64": 0.9911745064640191, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9970321077274646, + "pass@4": 0.9972259128097947, + "pass@8": 0.9975709158200651, + "pass@16": 0.9981634334351739, + "pass@32": 0.9990176819380228, + "pass@64": 0.9998121165453686, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.988973496835443, + "pass@2": 0.9914306226951061, + "pass@4": 0.993302117891909, + "pass@8": 0.9949012426776844, + "pass@16": 0.9961701906630579, + "pass@32": 0.9967511777435377, + "pass@64": 0.996835004987544, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9873076201201201, + "pass@2": 0.9915806554586083, + "pass@4": 0.9937272097744536, + "pass@8": 0.9952369199027994, + "pass@16": 0.996386500798402, + "pass@32": 0.9969175073660387, + "pass@64": 0.9969965813136217, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9869891225915893, + "pass@4": 0.9888716316007123, + "pass@8": 0.9903649875151894, + "pass@16": 0.9923085651126249, + "pass@32": 0.9943463657464062, + "pass@64": 0.9959421427097476, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.8953869047619047, + "pass@2": 0.9258264591926012, + "pass@4": 0.9429379149034938, + "pass@8": 0.9543336002203412, + "pass@16": 0.9626194265631142, + "pass@32": 0.9687413506580436, + "pass@64": 0.9737446542283149, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8249474388379205, + "pass@2": 0.8815236792361963, + "pass@4": 0.9135995612360381, + "pass@8": 0.9367215712258752, + "pass@16": 0.9539419568542121, + "pass@32": 0.9666327427155964, + "pass@64": 0.9768717219631041, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5698543595679012, + "pass@2": 0.6570694505200732, + "pass@4": 0.7250310091446903, + "pass@8": 0.7782503130285764, + "pass@16": 0.8207342556469278, + "pass@32": 0.8537480654369178, + "pass@64": 0.878043036531414, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.4921179525222552, + "pass@2": 0.5775018107899714, + "pass@4": 0.64585369929649, + "pass@8": 0.69798102049907, + "pass@16": 0.7376375558104614, + "pass@32": 0.770023465331237, + "pass@64": 0.7967844004152322, + "pass@128": 0.8219584569732937 + }, + "15": { + "pass@1": 0.3724922839506173, + "pass@2": 0.4515567281520361, + "pass@4": 0.5233148660236916, + "pass@8": 0.5866930510725332, + "pass@16": 0.6407629732953167, + "pass@32": 0.6878131369368291, + "pass@64": 0.7286931825352366, + "pass@128": 0.7623456790123457 + }, + "16": { + "pass@1": 0.24110435103244837, + "pass@2": 0.3068064362529904, + "pass@4": 0.3683298506381394, + "pass@8": 0.42239557551272405, + "pass@16": 0.4703387253798914, + "pass@32": 0.5155321182925883, + "pass@64": 0.5590083669683755, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.21724397590361447, + "pass@2": 0.2787211839483919, + "pass@4": 0.33454787795727337, + "pass@8": 0.38220498919307505, + "pass@16": 0.4241590468878052, + "pass@32": 0.4634047222244833, + "pass@64": 0.5003442497973006, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18961693548387096, + "pass@2": 0.2507727171704342, + "pass@4": 0.3110922971322133, + "pass@8": 0.3657547320819236, + "pass@16": 0.41192986664478526, + "pass@32": 0.45166304993993356, + "pass@64": 0.4878861429737296, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17265388972809667, + "pass@2": 0.22803200157004544, + "pass@4": 0.2792073111329362, + "pass@8": 0.32448211542126465, + "pass@16": 0.36277516546281496, + "pass@32": 0.39563085079972393, + "pass@64": 0.4277352297153177, + "pass@128": 0.4622356495468278 + }, + "20": { + "pass@1": 0.14379222972972974, + "pass@2": 0.1978464685157598, + "pass@4": 0.25417908262593303, + "pass@8": 0.310062539442377, + "pass@16": 0.3614706077644692, + "pass@32": 0.40526578602168234, + "pass@64": 0.44272531631551015, + "pass@128": 0.47147147147147145 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..994ae29fe9448d6ac4882f05faaeb91f41c7df00 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd3f721a0a649bb9c6c63e438c55a74dbdc295c8f353756808012ac240c44ac7 +size 2551963042 diff --git a/results/composition/SDAR-100M-block8-epoch1-op7-10/summary.csv b/results/composition/SDAR-100M-block8-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..c3271bc35d366d11411117e2e9c0d372ad4f6278 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op7-10,0.9847204861111111,0.40368125,0.6789103618421053,0.0,0.0,0.0,199.6684453125,320.694090625,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch1-op7-10/summary.json b/results/composition/SDAR-100M-block8-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe5fa505463dfdcf0183dae87482fbd08ebbbfb --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op7-10", + "id_acc": 0.9847204861111111, + "ood_acc": 0.40368125, + "total_acc": 0.6789103618421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.6684453125, + "ood_avg_resp_len": 320.694090625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 548183, + "answer_accuracy": 0.6769964728896617, + "avg_response_len": 251.36895599509958, + "resp_tokens_sum": 203540482, + "pass_at_k": { + "pass@1": 0.6769964728896617, + "pass@2": 0.7122231071742398, + "pass@4": 0.7411229584436191, + "pass@8": 0.7655143535115593, + "pass@16": 0.7864565900913838, + "pass@32": 0.8044771795508289, + "pass@64": 0.8204609794123039, + "pass@128": 0.8351248814416693 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9362555309734514, + "pass@2": 0.9539477504471235, + "pass@4": 0.9651495407653686, + "pass@8": 0.973229583190713, + "pass@16": 0.9792803436377407, + "pass@32": 0.9841718673769317, + "pass@64": 0.9878246376873651, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983753822629969, + "pass@2": 0.9995236762262515, + "pass@4": 0.9999511242287376, + "pass@8": 0.9999994411862023, + "pass@16": 0.9999999999675167, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990673828125, + "pass@2": 0.9933247723917322, + "pass@4": 0.9952177153637045, + "pass@8": 0.9963542387601867, + "pass@16": 0.9968111376105945, + "pass@32": 0.9968743549630685, + "pass@64": 0.9968749999981098, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9707093253968254, + "pass@2": 0.9767396262967132, + "pass@4": 0.981683998131186, + "pass@8": 0.9859164493385404, + "pass@16": 0.9901710891964574, + "pass@32": 0.9936881458577858, + "pass@64": 0.995845992043236, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9987512671507378, + "pass@4": 0.9995554992377451, + "pass@8": 0.999939295718634, + "pass@16": 0.9999991353050729, + "pass@32": 0.9999999999556102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958394970414202, + "pass@2": 0.9969697124120579, + "pass@4": 0.9970408646219519, + "pass@8": 0.9970414201059147, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9894080528846154, + "pass@2": 0.9919966434484152, + "pass@4": 0.9931632229865497, + "pass@8": 0.9938714043202131, + "pass@16": 0.9943410041176739, + "pass@32": 0.9949967186169365, + "pass@64": 0.9959998990510801, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9859595070422535, + "pass@2": 0.9918774952866809, + "pass@4": 0.9937267724456977, + "pass@8": 0.9954155716338281, + "pass@16": 0.9975424861780133, + "pass@32": 0.9993143415342073, + "pass@64": 0.9999593384604206, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8889872521246459, + "pass@2": 0.9209322232383845, + "pass@4": 0.9403707372980644, + "pass@8": 0.9523073338351158, + "pass@16": 0.9599911546995261, + "pass@32": 0.9656664426486973, + "pass@64": 0.97105654419183, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.807468220338983, + "pass@2": 0.8675287907602651, + "pass@4": 0.9025985565363646, + "pass@8": 0.9234312139724606, + "pass@16": 0.9362103618765187, + "pass@32": 0.9450079869218969, + "pass@64": 0.9525377320034726, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.6421923981191222, + "pass@2": 0.7228887178189717, + "pass@4": 0.7779022200986624, + "pass@8": 0.8182676410254136, + "pass@16": 0.8505300132234627, + "pass@32": 0.8773909089095185, + "pass@64": 0.9017227414682624, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.4485007788161994, + "pass@2": 0.5347201780852163, + "pass@4": 0.6061306235669136, + "pass@8": 0.6653760404004355, + "pass@16": 0.7137935405594372, + "pass@32": 0.7534356762985341, + "pass@64": 0.7870531477671298, + "pass@128": 0.8130841121495327 + }, + "15": { + "pass@1": 0.3333408598265896, + "pass@2": 0.41465276671521556, + "pass@4": 0.4854122544653017, + "pass@8": 0.5467994374688726, + "pass@16": 0.6020372401229079, + "pass@32": 0.6496721898260739, + "pass@64": 0.686791777517624, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2532258064516129, + "pass@2": 0.3180189547879094, + "pass@4": 0.374151785260713, + "pass@8": 0.42593478016771535, + "pass@16": 0.4749139463216263, + "pass@32": 0.5228389769740335, + "pass@64": 0.5721693101663297, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19495122832369943, + "pass@2": 0.2523230195484954, + "pass@4": 0.3023951223040761, + "pass@8": 0.3470740122741218, + "pass@16": 0.39059013486586935, + "pass@32": 0.4324684907446193, + "pass@64": 0.46828111323798854, + "pass@128": 0.49421965317919075 + }, + "18": { + "pass@1": 0.16796875, + "pass@2": 0.22441437007874018, + "pass@4": 0.28228091943864153, + "pass@8": 0.33871351937233285, + "pass@16": 0.38905522009524307, + "pass@32": 0.4321079296175608, + "pass@64": 0.47367519903850336, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1451998873873874, + "pass@2": 0.20028788631150843, + "pass@4": 0.2549552702308609, + "pass@8": 0.30319246623896356, + "pass@16": 0.34293555954657895, + "pass@32": 0.3751435020629878, + "pass@64": 0.4044465014859663, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.1495870870870871, + "pass@2": 0.20280615260930232, + "pass@4": 0.25683599347378866, + "pass@8": 0.30743826429217297, + "pass@16": 0.3517171120545327, + "pass@32": 0.38878767049319296, + "pass@64": 0.4209157334351972, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 554465, + "answer_accuracy": 0.6752545303975058, + "avg_response_len": 259.52538118667184, + "resp_tokens_sum": 213101481, + "pass_at_k": { + "pass@1": 0.6752545303975058, + "pass@2": 0.7099311637341081, + "pass@4": 0.7396101450831859, + "pass@8": 0.765588659194692, + "pass@16": 0.788283743412472, + "pass@32": 0.8079536752890428, + "pass@64": 0.8246852486176389, + "pass@128": 0.8385035074045206 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9209403757668712, + "pass@2": 0.9380713794019612, + "pass@4": 0.9512705088436028, + "pass@8": 0.9631955401424825, + "pass@16": 0.9726077613588666, + "pass@32": 0.9782359909847477, + "pass@64": 0.9813867257620852, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9994191422094966, + "pass@4": 0.9999210212359819, + "pass@8": 0.9999984042922265, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9975268121301775, + "pass@2": 0.9989032724921958, + "pass@4": 0.9997191858968812, + "pass@8": 0.9999779550101494, + "pass@16": 0.999999898523686, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875679347826087, + "pass@2": 0.9887766746547987, + "pass@4": 0.9894790216440336, + "pass@8": 0.9903547461693829, + "pass@16": 0.9916253749261505, + "pass@32": 0.9931572503524777, + "pass@64": 0.9949313065753761, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9943364971346705, + "pass@2": 0.9965329089863052, + "pass@4": 0.9980393601408706, + "pass@8": 0.9990811493323517, + "pass@16": 0.9997493490657728, + "pass@32": 0.9999854792814306, + "pass@64": 0.999999993573057, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934293713872833, + "pass@2": 0.9942072077966412, + "pass@4": 0.9943974088679668, + "pass@8": 0.9945809233687215, + "pass@16": 0.994942196531792, + "pass@32": 0.9956647398843931, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9836707746478873, + "pass@2": 0.987972371631363, + "pass@4": 0.9905331931219864, + "pass@8": 0.9926688812257355, + "pass@16": 0.9945746668596219, + "pass@32": 0.9961092494376016, + "pass@64": 0.9970112564626833, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9756034805389222, + "pass@2": 0.9805492361733227, + "pass@4": 0.9840381910532142, + "pass@8": 0.9872988101167206, + "pass@16": 0.9901482935355289, + "pass@32": 0.992455558127094, + "pass@64": 0.9937913521118914, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8997552710843374, + "pass@2": 0.9309293028412866, + "pass@4": 0.9496467379642303, + "pass@8": 0.9619478037609, + "pass@16": 0.9701481675058946, + "pass@32": 0.9759861111837409, + "pass@64": 0.981159773824385, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8052752742946708, + "pass@2": 0.8523317361093967, + "pass@4": 0.8786195395246442, + "pass@8": 0.8979108712169228, + "pass@16": 0.9138855036221462, + "pass@32": 0.9282479851031312, + "pass@64": 0.9436447705145927, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5835084033613446, + "pass@2": 0.6718991238227576, + "pass@4": 0.7351696928115078, + "pass@8": 0.7855705437580075, + "pass@16": 0.8282254349495101, + "pass@32": 0.8632778064510783, + "pass@64": 0.8906273038627206, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.4359466374269006, + "pass@2": 0.5169355372519222, + "pass@4": 0.5883138300146694, + "pass@8": 0.6484225858141512, + "pass@16": 0.6976308033859067, + "pass@32": 0.7384376268134236, + "pass@64": 0.7720516569697223, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.3393465909090909, + "pass@2": 0.4214902469577665, + "pass@4": 0.49189631068843637, + "pass@8": 0.5528083984205381, + "pass@16": 0.6063339208685132, + "pass@32": 0.6533444169232615, + "pass@64": 0.6909033146987849, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.22008547008547008, + "pass@2": 0.2828445022096597, + "pass@4": 0.34362964138029767, + "pass@8": 0.4025933786277824, + "pass@16": 0.45909006271494923, + "pass@32": 0.509012724160067, + "pass@64": 0.5514726875161375, + "pass@128": 0.5897435897435898 + }, + "17": { + "pass@1": 0.20479910714285715, + "pass@2": 0.26985353902772996, + "pass@4": 0.3339852589272611, + "pass@8": 0.39210840306068734, + "pass@16": 0.44226737100715613, + "pass@32": 0.4864598585876851, + "pass@64": 0.5269995343426999, + "pass@128": 0.562111801242236 + }, + "18": { + "pass@1": 0.18855698529411766, + "pass@2": 0.24998842056507656, + "pass@4": 0.30994159333024557, + "pass@8": 0.36606403192727205, + "pass@16": 0.4167227598892368, + "pass@32": 0.4612764576013937, + "pass@64": 0.4994967680463235, + "pass@128": 0.5294117647058824 + }, + "19": { + "pass@1": 0.1353701636904762, + "pass@2": 0.1830628105080615, + "pass@4": 0.23144565969432382, + "pass@8": 0.2755576222539975, + "pass@16": 0.3158109715954468, + "pass@32": 0.35526680353587153, + "pass@64": 0.39177456043247216, + "pass@128": 0.4226190476190476 + }, + "20": { + "pass@1": 0.15468282185628743, + "pass@2": 0.21383758074402373, + "pass@4": 0.2715573271342579, + "pass@8": 0.32506282832444283, + "pass@16": 0.37242304719707847, + "pass@32": 0.41287302118538943, + "pass@64": 0.4455683819577822, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 548462, + "answer_accuracy": 0.6845916879693241, + "avg_response_len": 279.4282757828727, + "resp_tokens_sum": 223864522, + "pass_at_k": { + "pass@1": 0.6845916879693241, + "pass@2": 0.7197870499551496, + "pass@4": 0.7493415314717663, + "pass@8": 0.7743609564571847, + "pass@16": 0.7953491263112427, + "pass@32": 0.8132410231819698, + "pass@64": 0.8289152032459771, + "pass@128": 0.8429461575331523 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9393889925373134, + "pass@2": 0.9574362439769657, + "pass@4": 0.9697268340524599, + "pass@8": 0.9789975497663034, + "pass@16": 0.9853411276247855, + "pass@32": 0.9895658032002611, + "pass@64": 0.9921389084495922, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99953125, + "pass@2": 0.9999261811023622, + "pass@4": 0.9999983970753656, + "pass@8": 0.9999999995933421, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999763263010492, + "pass@4": 0.9999999330440837, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969618055555556, + "pass@2": 0.9970745844269466, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9802849264705882, + "pass@2": 0.984084790412228, + "pass@4": 0.9867516963872165, + "pass@8": 0.9888766116409934, + "pass@16": 0.9903040195148527, + "pass@32": 0.9910264810308619, + "pass@64": 0.9911745064640191, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9970321077274646, + "pass@4": 0.9972259128097947, + "pass@8": 0.9975709158200651, + "pass@16": 0.9981634334351739, + "pass@32": 0.9990176819380228, + "pass@64": 0.9998121165453686, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.988973496835443, + "pass@2": 0.9914306226951061, + "pass@4": 0.993302117891909, + "pass@8": 0.9949012426776844, + "pass@16": 0.9961701906630579, + "pass@32": 0.9967511777435377, + "pass@64": 0.996835004987544, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9873076201201201, + "pass@2": 0.9915806554586083, + "pass@4": 0.9937272097744536, + "pass@8": 0.9952369199027994, + "pass@16": 0.996386500798402, + "pass@32": 0.9969175073660387, + "pass@64": 0.9969965813136217, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9869891225915893, + "pass@4": 0.9888716316007123, + "pass@8": 0.9903649875151894, + "pass@16": 0.9923085651126249, + "pass@32": 0.9943463657464062, + "pass@64": 0.9959421427097476, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.8953869047619047, + "pass@2": 0.9258264591926012, + "pass@4": 0.9429379149034938, + "pass@8": 0.9543336002203412, + "pass@16": 0.9626194265631142, + "pass@32": 0.9687413506580436, + "pass@64": 0.9737446542283149, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8249474388379205, + "pass@2": 0.8815236792361963, + "pass@4": 0.9135995612360381, + "pass@8": 0.9367215712258752, + "pass@16": 0.9539419568542121, + "pass@32": 0.9666327427155964, + "pass@64": 0.9768717219631041, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5698543595679012, + "pass@2": 0.6570694505200732, + "pass@4": 0.7250310091446903, + "pass@8": 0.7782503130285764, + "pass@16": 0.8207342556469278, + "pass@32": 0.8537480654369178, + "pass@64": 0.878043036531414, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.4921179525222552, + "pass@2": 0.5775018107899714, + "pass@4": 0.64585369929649, + "pass@8": 0.69798102049907, + "pass@16": 0.7376375558104614, + "pass@32": 0.770023465331237, + "pass@64": 0.7967844004152322, + "pass@128": 0.8219584569732937 + }, + "15": { + "pass@1": 0.3724922839506173, + "pass@2": 0.4515567281520361, + "pass@4": 0.5233148660236916, + "pass@8": 0.5866930510725332, + "pass@16": 0.6407629732953167, + "pass@32": 0.6878131369368291, + "pass@64": 0.7286931825352366, + "pass@128": 0.7623456790123457 + }, + "16": { + "pass@1": 0.24110435103244837, + "pass@2": 0.3068064362529904, + "pass@4": 0.3683298506381394, + "pass@8": 0.42239557551272405, + "pass@16": 0.4703387253798914, + "pass@32": 0.5155321182925883, + "pass@64": 0.5590083669683755, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.21724397590361447, + "pass@2": 0.2787211839483919, + "pass@4": 0.33454787795727337, + "pass@8": 0.38220498919307505, + "pass@16": 0.4241590468878052, + "pass@32": 0.4634047222244833, + "pass@64": 0.5003442497973006, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18961693548387096, + "pass@2": 0.2507727171704342, + "pass@4": 0.3110922971322133, + "pass@8": 0.3657547320819236, + "pass@16": 0.41192986664478526, + "pass@32": 0.45166304993993356, + "pass@64": 0.4878861429737296, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17265388972809667, + "pass@2": 0.22803200157004544, + "pass@4": 0.2792073111329362, + "pass@8": 0.32448211542126465, + "pass@16": 0.36277516546281496, + "pass@32": 0.39563085079972393, + "pass@64": 0.4277352297153177, + "pass@128": 0.4622356495468278 + }, + "20": { + "pass@1": 0.14379222972972974, + "pass@2": 0.1978464685157598, + "pass@4": 0.25417908262593303, + "pass@8": 0.310062539442377, + "pass@16": 0.3614706077644692, + "pass@32": 0.40526578602168234, + "pass@64": 0.44272531631551015, + "pass@128": 0.47147147147147145 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3897b159bb65fc97dffa726b566a93b8806ff377 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:375b3aa088012be5464399be01c62fe3839763ed2f09d881cefb7a5b3140d6bb +size 1790650294 diff --git a/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json b/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a8f4296d339151d1b2a55dc3a4343c2c49fa8df0 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9833871527777778, + "avg_response_len": 198.9335685763889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9833871527777778, + "pass@2": 0.9877975311679758, + "pass@4": 0.9904131950172896, + "pass@8": 0.9922471730307514, + "pass@16": 0.9938091197969837, + "pass@32": 0.9952691718510881, + "pass@64": 0.9965101656296633, + "pass@128": 0.9973333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.936140625, + "pass@2": 0.9532770669291335, + "pass@4": 0.9646779643794529, + "pass@8": 0.9722428738775302, + "pass@16": 0.9780534399094882, + "pass@32": 0.9829244527998198, + "pass@64": 0.9868526799974783, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999421875, + "pass@2": 0.9997268700787402, + "pass@4": 0.9999281513873266, + "pass@8": 0.9999954379593903, + "pass@16": 0.999999987960665, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.9982974901574803, + "pass@4": 0.9992196539182601, + "pass@8": 0.9997844261478821, + "pass@16": 0.999981904078783, + "pass@32": 0.9999999207944156, + "pass@64": 0.9999999999999699, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9924765625, + "pass@2": 0.9934320866141733, + "pass@4": 0.9943405821147356, + "pass@8": 0.9951957912852285, + "pass@16": 0.9958488669003491, + "pass@32": 0.9965198504237649, + "pass@64": 0.9973183937001826, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9787734375, + "pass@2": 0.9831724901574804, + "pass@4": 0.9860739181664795, + "pass@8": 0.9882095975142469, + "pass@16": 0.9906008995622421, + "pass@32": 0.993477161814902, + "pass@64": 0.9959503664688322, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9934375, + "pass@2": 0.9951104822834648, + "pass@4": 0.9958734730971127, + "pass@8": 0.9965130401102446, + "pass@16": 0.9972419328539005, + "pass@32": 0.9980359313833913, + "pass@64": 0.9987493335625536, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.99075, + "pass@2": 0.9936044537401577, + "pass@4": 0.994668941319835, + "pass@8": 0.9952967222076122, + "pass@16": 0.9959389380230231, + "pass@32": 0.9967077687453606, + "pass@64": 0.997570362137923, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.983, + "pass@2": 0.9881865157480314, + "pass@4": 0.9911749555680541, + "pass@8": 0.9933137034852952, + "pass@16": 0.9948246809075276, + "pass@32": 0.9958187164772767, + "pass@64": 0.9964386629519557, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.979484375, + "pass@2": 0.9853703248031499, + "pass@4": 0.9877611152043493, + "pass@8": 0.9896729646893262, + "pass@16": 0.9917914279768701, + "pass@32": 0.993938744220871, + "pass@64": 0.9957116918480703, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.936140625, + "2": 0.999421875, + "3": 0.997, + "4": 0.9924765625, + "5": 0.9787734375, + "6": 0.9934375, + "7": 0.99075, + "8": 0.983, + "9": 0.979484375 + }, + "per_op_avg_response_len": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 372515, + "answer_accuracy": 0.9828684354947653, + "avg_response_len": 189.65915495187437, + "resp_tokens_sum": 71882337, + "pass_at_k": { + "pass@1": 0.9828684354947653, + "pass@2": 0.9877498690323291, + "pass@4": 0.990514759869809, + "pass@8": 0.9923264350093163, + "pass@16": 0.9938964450532994, + "pass@32": 0.9954374275388409, + "pass@64": 0.9967093195612354, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9386292404129793, + "pass@2": 0.9558454840545374, + "pass@4": 0.9661186383781674, + "pass@8": 0.9715414375472532, + "pass@16": 0.9756175864300959, + "pass@32": 0.9796003458610456, + "pass@64": 0.9837013134257441, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977780963302753, + "pass@2": 0.9989322220616917, + "pass@4": 0.9996694776685024, + "pass@8": 0.9999677623799305, + "pass@16": 0.999999779641568, + "pass@32": 0.9999999999981484, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837158203125, + "pass@2": 0.9857337290846455, + "pass@4": 0.9882842340996436, + "pass@8": 0.9906827001964222, + "pass@16": 0.9922391181843271, + "pass@32": 0.9935574745502678, + "pass@64": 0.9951263779508654, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9695188492063492, + "pass@2": 0.976019013248344, + "pass@4": 0.9805692556287607, + "pass@8": 0.9843784633909355, + "pass@16": 0.9889107918312874, + "pass@32": 0.9940484172603424, + "pass@64": 0.9981441113782779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9948774326347305, + "pass@2": 0.9966545747088502, + "pass@4": 0.9974962398038569, + "pass@8": 0.9981117754532004, + "pass@16": 0.9988525144880754, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9937361316568047, + "pass@2": 0.9963439989749802, + "pass@4": 0.9969306777666104, + "pass@8": 0.9970380005155915, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9876302083333334, + "pass@2": 0.9916015230668282, + "pass@4": 0.9930404943372461, + "pass@8": 0.9940772665714049, + "pass@16": 0.9949131147821092, + "pass@32": 0.9957967829765609, + "pass@64": 0.9966039773874414, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9811399647887324, + "pass@2": 0.9891198569368971, + "pass@4": 0.9927404183631975, + "pass@8": 0.9952636734053902, + "pass@16": 0.9975775655877893, + "pass@32": 0.999379044270334, + "pass@64": 0.9999770279396809, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 384130, + "answer_accuracy": 0.9832947657273918, + "avg_response_len": 196.8890993610747, + "resp_tokens_sum": 76915908, + "pass_at_k": { + "pass@1": 0.9832947657273918, + "pass@2": 0.9873143320889366, + "pass@4": 0.9897567537941279, + "pass@8": 0.9917119681000646, + "pass@16": 0.9935082848685695, + "pass@32": 0.9950803838021421, + "pass@64": 0.9962572852675725, + "pass@128": 0.9967234600262124 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312691717791411, + "pass@2": 0.9492200708902958, + "pass@4": 0.9614929713540408, + "pass@8": 0.9706849500303524, + "pass@16": 0.9780743583738936, + "pass@32": 0.9833452492500756, + "pass@64": 0.9865077334207035, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961647727272728, + "pass@2": 0.9978696910045335, + "pass@4": 0.9990399083637271, + "pass@8": 0.9997415688663096, + "pass@16": 0.9999820837105999, + "pass@32": 0.9999999593969077, + "pass@64": 0.9999999999999987, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963480029585798, + "pass@2": 0.9970392361272888, + "pass@4": 0.9973065910215366, + "pass@8": 0.9975660568709271, + "pass@16": 0.9980247014241554, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9867753623188406, + "pass@2": 0.9882117996120052, + "pass@4": 0.9893897974709683, + "pass@8": 0.9906458350692433, + "pass@16": 0.9923677751658371, + "pass@32": 0.9946202004072046, + "pass@64": 0.996556005154711, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9901504297994269, + "pass@2": 0.9921238690973083, + "pass@4": 0.9931829475828413, + "pass@8": 0.9941730137747032, + "pass@16": 0.9951084353395996, + "pass@32": 0.9959328579290595, + "pass@64": 0.9967849648397139, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9909230491329479, + "pass@2": 0.9937133039005962, + "pass@4": 0.9946745414951744, + "pass@8": 0.9952348105100473, + "pass@16": 0.9961070653038383, + "pass@32": 0.9974785593997282, + "pass@64": 0.9991110099676848, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840008802816902, + "pass@2": 0.9890027170899413, + "pass@4": 0.9920841439714402, + "pass@8": 0.9944509310383022, + "pass@16": 0.9961687806906195, + "pass@32": 0.9970422143124665, + "pass@64": 0.9971821603034191, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.977356311000047, + "pass@4": 0.9796722011544962, + "pass@8": 0.9821828386883638, + "pass@16": 0.9851973646020666, + "pass@32": 0.988117457332326, + "pass@64": 0.9901673949748785, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 376217, + "answer_accuracy": 0.9839957524271845, + "avg_response_len": 210.2162129645129, + "resp_tokens_sum": 80373226, + "pass_at_k": { + "pass@1": 0.9839957524271845, + "pass@2": 0.9883384923909123, + "pass@4": 0.9909832402415216, + "pass@8": 0.9927154524850256, + "pass@16": 0.9940299360398915, + "pass@32": 0.9952952769846484, + "pass@64": 0.9965711284933091, + "pass@128": 0.9976565115500502 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.9546259842519684, + "pass@4": 0.9663195143517505, + "pass@8": 0.9744687547436317, + "pass@16": 0.9804980217307362, + "pass@32": 0.9858787590967183, + "pass@64": 0.9903773541820934, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9984375, + "pass@2": 0.9992217379077615, + "pass@4": 0.9997947185173283, + "pass@8": 0.999986965598258, + "pass@16": 0.9999999656019002, + "pass@32": 0.999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970617711370262, + "pass@2": 0.9981039519294782, + "pass@4": 0.9989637462409036, + "pass@8": 0.9996508749963928, + "pass@16": 0.9999646895378788, + "pass@32": 0.9999998081458937, + "pass@64": 0.9999999999999133, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968475877192983, + "pass@2": 0.997070267532348, + "pass@4": 0.9970760217472814, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9792279411764706, + "pass@2": 0.9846865591709126, + "pass@4": 0.9878091838703983, + "pass@8": 0.9892869660212149, + "pass@16": 0.9903738755975673, + "pass@32": 0.9917880624629651, + "pass@64": 0.9933033811949977, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9955392350157729, + "pass@2": 0.9967716845922651, + "pass@4": 0.9971258053508296, + "pass@8": 0.9974048116766691, + "pass@16": 0.997893845682534, + "pass@32": 0.9986796800242242, + "pass@64": 0.9996149929208377, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9873664952531646, + "pass@2": 0.9905549966361007, + "pass@4": 0.9922436103714881, + "pass@8": 0.9935020050533728, + "pass@16": 0.9945756140766189, + "pass@32": 0.995506921497008, + "pass@64": 0.996449217370583, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9775947822822822, + "pass@2": 0.9841167446974534, + "pass@4": 0.9884578084833988, + "pass@8": 0.9913859333225822, + "pass@16": 0.9933089247757945, + "pass@32": 0.9945349372602522, + "pass@64": 0.995491156454535, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9852290996784566, + "pass@2": 0.9896970023039725, + "pass@4": 0.9907644742759243, + "pass@8": 0.9913353456864922, + "pass@16": 0.9922684000518165, + "pass@32": 0.9939805554852268, + "pass@64": 0.9967972251057042, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.51230234375, + "avg_response_len": 344.525071875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.51230234375, + "pass@2": 0.5791796751968412, + "pass@4": 0.6338488173040775, + "pass@8": 0.6792057747266516, + "pass@16": 0.7175458302412239, + "pass@32": 0.7502231938070137, + "pass@64": 0.7779405120031644, + "pass@128": 0.8017 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.922875, + "pass@2": 0.94406471456693, + "pass@4": 0.9561565563367079, + "pass@8": 0.9644606428754197, + "pass@16": 0.9710626466016383, + "pass@32": 0.9765538580830442, + "pass@64": 0.9809558784585826, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.8785, + "pass@2": 0.9094697342519692, + "pass@4": 0.9260017212223469, + "pass@8": 0.9373513202159709, + "pass@16": 0.9468305077816318, + "pass@32": 0.9552749829016273, + "pass@64": 0.961888979167555, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.747625, + "pass@2": 0.8090279281496071, + "pass@4": 0.8502927079115105, + "pass@8": 0.8808873518388468, + "pass@16": 0.9045519413705413, + "pass@32": 0.9237040116121091, + "pass@64": 0.9397421870047707, + "pass@128": 0.953 + }, + "14": { + "pass@1": 0.662765625, + "pass@2": 0.7385823080708667, + "pass@4": 0.7916132981814766, + "pass@8": 0.8299020217310532, + "pass@16": 0.858302743335784, + "pass@32": 0.8802205162431695, + "pass@64": 0.8972672936992975, + "pass@128": 0.912 + }, + "15": { + "pass@1": 0.5640078125, + "pass@2": 0.6509618602362214, + "pass@4": 0.7146336797900266, + "pass@8": 0.7634876908837895, + "pass@16": 0.8021745427082866, + "pass@32": 0.8324330314086336, + "pass@64": 0.8563183762197656, + "pass@128": 0.878 + }, + "16": { + "pass@1": 0.4091171875, + "pass@2": 0.503646161417323, + "pass@4": 0.5834662753093354, + "pass@8": 0.6506320323760373, + "pass@16": 0.7081603961156945, + "pass@32": 0.7557823960324996, + "pass@64": 0.7951181687007922, + "pass@128": 0.831 + }, + "17": { + "pass@1": 0.30865625, + "pass@2": 0.3904856053149604, + "pass@4": 0.4633976213910753, + "pass@8": 0.5266492028932148, + "pass@16": 0.5819307639208975, + "pass@32": 0.6299621389073284, + "pass@64": 0.6725238032552022, + "pass@128": 0.709 + }, + "18": { + "pass@1": 0.2607265625, + "pass@2": 0.34324064960629913, + "pass@4": 0.4184819655043119, + "pass@8": 0.4815590749232352, + "pass@16": 0.5333000164446824, + "pass@32": 0.5786741441482469, + "pass@64": 0.6197798116311323, + "pass@128": 0.656 + }, + "19": { + "pass@1": 0.18853125, + "pass@2": 0.25494758858267685, + "pass@4": 0.31918568944506914, + "pass@8": 0.3778321490296506, + "pass@16": 0.4313708711520126, + "pass@32": 0.4797498884371214, + "pass@64": 0.5224194950656782, + "pass@128": 0.562 + }, + "20": { + "pass@1": 0.18021875, + "pass@2": 0.24737020177165342, + "pass@4": 0.3152586579490061, + "pass@8": 0.3792962604993045, + "pass@16": 0.4377738729809448, + "pass@32": 0.48987697029632693, + "pass@64": 0.5333911268287946, + "pass@128": 0.567 + } + }, + "per_op_accuracy": { + "11": 0.922875, + "12": 0.8785, + "13": 0.747625, + "14": 0.662765625, + "15": 0.5640078125, + "16": 0.4091171875, + "17": 0.30865625, + "18": 0.2607265625, + "19": 0.18853125, + "20": 0.18021875 + }, + "per_op_avg_response_len": { + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 221157, + "answer_accuracy": 0.5134588595839524, + "avg_response_len": 327.17834323922733, + "resp_tokens_sum": 140922256, + "pass_at_k": { + "pass@1": 0.5134588595839524, + "pass@2": 0.5795858390565243, + "pass@4": 0.6329100493010436, + "pass@8": 0.6767816699546235, + "pass@16": 0.7142134037015744, + "pass@32": 0.746710897314702, + "pass@64": 0.7742172018805795, + "pass@128": 0.7976225854383359 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9211225212464589, + "pass@2": 0.9423424918025475, + "pass@4": 0.953929790822889, + "pass@8": 0.9614565186188683, + "pass@16": 0.9677885340638542, + "pass@32": 0.9743190927138413, + "pass@64": 0.9805943394845731, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8815986935028248, + "pass@2": 0.9165651830597449, + "pass@4": 0.934161171855637, + "pass@8": 0.9452938352373702, + "pass@16": 0.9539612829997371, + "pass@32": 0.9615647682922526, + "pass@64": 0.9668943055418103, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7734619905956113, + "pass@2": 0.8301158732999284, + "pass@4": 0.8655036610823328, + "pass@8": 0.8898255448648787, + "pass@16": 0.9097976575302069, + "pass@32": 0.92890026616942, + "pass@64": 0.9464289475637889, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.6513093847352025, + "pass@2": 0.7309117208771797, + "pass@4": 0.7865110567370666, + "pass@8": 0.8270123821269293, + "pass@16": 0.8587307099359938, + "pass@32": 0.8847289386929739, + "pass@64": 0.9048481484220781, + "pass@128": 0.9221183800623053 + }, + "15": { + "pass@1": 0.5636741329479769, + "pass@2": 0.6490562133039006, + "pass@4": 0.7107771526752794, + "pass@8": 0.7595685140977119, + "pass@16": 0.8003622070803227, + "pass@32": 0.8331029501088946, + "pass@64": 0.8575529239777137, + "pass@128": 0.8757225433526011 + }, + "16": { + "pass@1": 0.4171118951612903, + "pass@2": 0.5142795910591821, + "pass@4": 0.5949670703460455, + "pass@8": 0.6603091766742203, + "pass@16": 0.7134515965927327, + "pass@32": 0.75739935199652, + "pass@64": 0.7984302135889665, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.31419346098265893, + "pass@2": 0.3958058349642712, + "pass@4": 0.46747656868035986, + "pass@8": 0.5300846478128985, + "pass@16": 0.585240494400111, + "pass@32": 0.6313648084038579, + "pass@64": 0.669614375035597, + "pass@128": 0.6994219653179191 + }, + "18": { + "pass@1": 0.22939732142857142, + "pass@2": 0.3074957817772778, + "pass@4": 0.38013197760994144, + "pass@8": 0.44242261962567625, + "pass@16": 0.49595147536216044, + "pass@32": 0.5439121161703538, + "pass@64": 0.5859423405040576, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18125938438438438, + "pass@2": 0.24487775176751556, + "pass@4": 0.3061006349431548, + "pass@8": 0.36073586559610316, + "pass@16": 0.40994189428993016, + "pass@32": 0.45423626681010243, + "pass@64": 0.4932719107305281, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.18332394894894896, + "pass@2": 0.25119817159679353, + "pass@4": 0.3202794138345321, + "pass@8": 0.3847688536744722, + "pass@16": 0.4427514993287269, + "pass@32": 0.4952158151961868, + "pass@64": 0.5381104843576482, + "pass@128": 0.5675675675675675 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 220408, + "answer_accuracy": 0.5120242343146001, + "avg_response_len": 339.50148444469227, + "resp_tokens_sum": 146143167, + "pass_at_k": { + "pass@1": 0.5120242343146001, + "pass@2": 0.5796171090093472, + "pass@4": 0.6347312556603939, + "pass@8": 0.6796315668175029, + "pass@16": 0.7166588115324342, + "pass@32": 0.7480615469732743, + "pass@64": 0.7755778775018244, + "pass@128": 0.7995837050252751 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.928699171686747, + "pass@2": 0.9515493074660848, + "pass@4": 0.9659594759804117, + "pass@8": 0.9766550766070073, + "pass@16": 0.9842963536346357, + "pass@32": 0.9884709338432213, + "pass@64": 0.9904158337215808, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8799960815047022, + "pass@2": 0.9035926739565081, + "pass@4": 0.9153743348695834, + "pass@8": 0.9229959569636945, + "pass@16": 0.9296964586106998, + "pass@32": 0.9365110569119893, + "pass@64": 0.94190277023684, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.7331276260504201, + "pass@2": 0.7954751290279893, + "pass@4": 0.8377238864049549, + "pass@8": 0.8699515828594602, + "pass@16": 0.8957292705806627, + "pass@32": 0.9165216166375357, + "pass@64": 0.9327622224638712, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6675804093567251, + "pass@2": 0.7419245291706962, + "pass@4": 0.796447699794104, + "pass@8": 0.8363445657414356, + "pass@16": 0.8626242889188614, + "pass@32": 0.8793508082194177, + "pass@64": 0.890769595219482, + "pass@128": 0.8976608187134503 + }, + "15": { + "pass@1": 0.5677320075757576, + "pass@2": 0.6639912908613691, + "pass@4": 0.7317696557816629, + "pass@8": 0.7795323520517282, + "pass@16": 0.8155345673543264, + "pass@32": 0.845208030617802, + "pass@64": 0.8714240604997285, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4091880341880342, + "pass@2": 0.501739616281939, + "pass@4": 0.5781482055661846, + "pass@8": 0.6406309064652197, + "pass@16": 0.6934298295552146, + "pass@32": 0.7373148005034644, + "pass@64": 0.7740434816672218, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3116022903726708, + "pass@2": 0.39830262385680026, + "pass@4": 0.47726986174476643, + "pass@8": 0.5445244698438653, + "pass@16": 0.5995497047481053, + "pass@32": 0.6445440362273804, + "pass@64": 0.6840008853131072, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.2716911764705882, + "pass@2": 0.35394171201945335, + "pass@4": 0.4276034548071196, + "pass@8": 0.4888071795579154, + "pass@16": 0.5390270190145257, + "pass@32": 0.5858273452671591, + "pass@64": 0.6344610738799759, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.17631603422619047, + "pass@2": 0.24067853334739409, + "pass@4": 0.3043591830597069, + "pass@8": 0.36333850531952205, + "pass@16": 0.41840949299425007, + "pass@32": 0.4696348856330499, + "pass@64": 0.5155865519395338, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.1794068113772455, + "pass@2": 0.24754415872978455, + "pass@4": 0.31347590112238966, + "pass@8": 0.37303300737954526, + "pass@16": 0.4267697863227694, + "pass@32": 0.4749624791026021, + "pass@64": 0.5176129128175474, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 214182, + "answer_accuracy": 0.5113988004278729, + "avg_response_len": 367.52814839929704, + "resp_tokens_sum": 153926669, + "pass_at_k": { + "pass@1": 0.5113988004278729, + "pass@2": 0.5783123673425716, + "pass@4": 0.6339072873951591, + "pass@8": 0.68126114567909, + "pass@16": 0.721884662522243, + "pass@32": 0.7560570862270739, + "pass@64": 0.7841979931738258, + "pass@128": 0.8080684596577017 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9187003968253968, + "pass@2": 0.9381061742282217, + "pass@4": 0.9483199814308924, + "pass@8": 0.9549746233950248, + "pass@16": 0.9607838243504727, + "pass@32": 0.9664979946638379, + "pass@64": 0.9713905391268673, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8736859709480123, + "pass@2": 0.9075217017024249, + "pass@4": 0.927535943614846, + "pass@8": 0.942757162906802, + "pass@16": 0.9558258204982006, + "pass@32": 0.9667707271291923, + "pass@64": 0.9759676186550527, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7381606867283951, + "pass@2": 0.8031985910615335, + "pass@4": 0.8491654709249306, + "pass@8": 0.884136691191738, + "pass@16": 0.9091084537688574, + "pass@32": 0.9265018813718013, + "pass@64": 0.9408495040503696, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6687917284866469, + "pass@2": 0.742496904133274, + "pass@4": 0.7915671680691242, + "pass@8": 0.8261163370467007, + "pass@16": 0.8535094321545398, + "pass@32": 0.8768087552275502, + "pass@64": 0.8966404643642354, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.560570987654321, + "pass@2": 0.6397261865704283, + "pass@4": 0.7012987609766366, + "pass@8": 0.7513312000892324, + "pass@16": 0.7905025056529893, + "pass@32": 0.8187060511329679, + "pass@64": 0.8396145819708817, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.4017330383480826, + "pass@2": 0.4958964096578634, + "pass@4": 0.5784556440953726, + "pass@8": 0.6521378744476022, + "pass@16": 0.7185738377524138, + "pass@32": 0.7734250794597738, + "pass@64": 0.8139101487404639, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.3000282379518072, + "pass@2": 0.3773594624561235, + "pass@4": 0.4456922685116164, + "pass@8": 0.5057320345187564, + "pass@16": 0.561393216655332, + "pass@32": 0.6143576371517395, + "pass@64": 0.6644245313917596, + "pass@128": 0.7108433734939759 + }, + "18": { + "pass@1": 0.2840725806451613, + "pass@2": 0.37186110934721867, + "pass@4": 0.4517761248593924, + "pass@8": 0.5177958613050213, + "pass@16": 0.5691864954935101, + "pass@32": 0.6100761487025451, + "pass@64": 0.6418813785016764, + "pass@128": 0.667741935483871 + }, + "19": { + "pass@1": 0.2082467900302115, + "pass@2": 0.27956287020957715, + "pass@4": 0.3474002794590251, + "pass@8": 0.4097443141957358, + "pass@16": 0.4660864976054333, + "pass@32": 0.5156854683282558, + "pass@64": 0.5586793575248601, + "pass@128": 0.595166163141994 + }, + "20": { + "pass@1": 0.17792792792792791, + "pass@2": 0.24336775259511484, + "pass@4": 0.3120260125142014, + "pass@8": 0.38010572901182366, + "pass@16": 0.44383337859661776, + "pass@32": 0.499497404852035, + "pass@64": 0.5444973653352457, + "pass@128": 0.5795795795795796 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7354477796052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7354477796052632, + "pass@2": 0.7727355017094696, + "pass@4": 0.8027477330629547, + "pass@8": 0.8274885423443955, + "pass@16": 0.8484073884518488, + "pass@32": 0.8662976044594782, + "pass@64": 0.8814735058262421, + "pass@128": 0.8943684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.936140625, + "pass@2": 0.9532770669291335, + "pass@4": 0.9646779643794529, + "pass@8": 0.9722428738775302, + "pass@16": 0.9780534399094882, + "pass@32": 0.9829244527998198, + "pass@64": 0.9868526799974783, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999421875, + "pass@2": 0.9997268700787402, + "pass@4": 0.9999281513873266, + "pass@8": 0.9999954379593903, + "pass@16": 0.999999987960665, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.9982974901574803, + "pass@4": 0.9992196539182601, + "pass@8": 0.9997844261478821, + "pass@16": 0.999981904078783, + "pass@32": 0.9999999207944156, + "pass@64": 0.9999999999999699, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9924765625, + "pass@2": 0.9934320866141733, + "pass@4": 0.9943405821147356, + "pass@8": 0.9951957912852285, + "pass@16": 0.9958488669003491, + "pass@32": 0.9965198504237649, + "pass@64": 0.9973183937001826, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9787734375, + "pass@2": 0.9831724901574804, + "pass@4": 0.9860739181664795, + "pass@8": 0.9882095975142469, + "pass@16": 0.9906008995622421, + "pass@32": 0.993477161814902, + "pass@64": 0.9959503664688322, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9934375, + "pass@2": 0.9951104822834648, + "pass@4": 0.9958734730971127, + "pass@8": 0.9965130401102446, + "pass@16": 0.9972419328539005, + "pass@32": 0.9980359313833913, + "pass@64": 0.9987493335625536, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.99075, + "pass@2": 0.9936044537401577, + "pass@4": 0.994668941319835, + "pass@8": 0.9952967222076122, + "pass@16": 0.9959389380230231, + "pass@32": 0.9967077687453606, + "pass@64": 0.997570362137923, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.983, + "pass@2": 0.9881865157480314, + "pass@4": 0.9911749555680541, + "pass@8": 0.9933137034852952, + "pass@16": 0.9948246809075276, + "pass@32": 0.9958187164772767, + "pass@64": 0.9964386629519557, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.979484375, + "pass@2": 0.9853703248031499, + "pass@4": 0.9877611152043493, + "pass@8": 0.9896729646893262, + "pass@16": 0.9917914279768701, + "pass@32": 0.993938744220871, + "pass@64": 0.9957116918480703, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.922875, + "pass@2": 0.94406471456693, + "pass@4": 0.9561565563367079, + "pass@8": 0.9644606428754197, + "pass@16": 0.9710626466016383, + "pass@32": 0.9765538580830442, + "pass@64": 0.9809558784585826, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.8785, + "pass@2": 0.9094697342519692, + "pass@4": 0.9260017212223469, + "pass@8": 0.9373513202159709, + "pass@16": 0.9468305077816318, + "pass@32": 0.9552749829016273, + "pass@64": 0.961888979167555, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.747625, + "pass@2": 0.8090279281496071, + "pass@4": 0.8502927079115105, + "pass@8": 0.8808873518388468, + "pass@16": 0.9045519413705413, + "pass@32": 0.9237040116121091, + "pass@64": 0.9397421870047707, + "pass@128": 0.953 + }, + "14": { + "pass@1": 0.662765625, + "pass@2": 0.7385823080708667, + "pass@4": 0.7916132981814766, + "pass@8": 0.8299020217310532, + "pass@16": 0.858302743335784, + "pass@32": 0.8802205162431695, + "pass@64": 0.8972672936992975, + "pass@128": 0.912 + }, + "15": { + "pass@1": 0.5640078125, + "pass@2": 0.6509618602362214, + "pass@4": 0.7146336797900266, + "pass@8": 0.7634876908837895, + "pass@16": 0.8021745427082866, + "pass@32": 0.8324330314086336, + "pass@64": 0.8563183762197656, + "pass@128": 0.878 + }, + "16": { + "pass@1": 0.4091171875, + "pass@2": 0.503646161417323, + "pass@4": 0.5834662753093354, + "pass@8": 0.6506320323760373, + "pass@16": 0.7081603961156945, + "pass@32": 0.7557823960324996, + "pass@64": 0.7951181687007922, + "pass@128": 0.831 + }, + "17": { + "pass@1": 0.30865625, + "pass@2": 0.3904856053149604, + "pass@4": 0.4633976213910753, + "pass@8": 0.5266492028932148, + "pass@16": 0.5819307639208975, + "pass@32": 0.6299621389073284, + "pass@64": 0.6725238032552022, + "pass@128": 0.709 + }, + "18": { + "pass@1": 0.2607265625, + "pass@2": 0.34324064960629913, + "pass@4": 0.4184819655043119, + "pass@8": 0.4815590749232352, + "pass@16": 0.5333000164446824, + "pass@32": 0.5786741441482469, + "pass@64": 0.6197798116311323, + "pass@128": 0.656 + }, + "19": { + "pass@1": 0.18853125, + "pass@2": 0.25494758858267685, + "pass@4": 0.31918568944506914, + "pass@8": 0.3778321490296506, + "pass@16": 0.4313708711520126, + "pass@32": 0.4797498884371214, + "pass@64": 0.5224194950656782, + "pass@128": 0.562 + }, + "20": { + "pass@1": 0.18021875, + "pass@2": 0.24737020177165342, + "pass@4": 0.3152586579490061, + "pass@8": 0.3792962604993045, + "pass@16": 0.4377738729809448, + "pass@32": 0.48987697029632693, + "pass@64": 0.5333911268287946, + "pass@128": 0.567 + } + }, + "per_op_accuracy": { + "10": 0.936140625, + "2": 0.999421875, + "3": 0.997, + "4": 0.9924765625, + "5": 0.9787734375, + "6": 0.9934375, + "7": 0.99075, + "8": 0.983, + "9": 0.979484375, + "11": 0.922875, + "12": 0.8785, + "13": 0.747625, + "14": 0.662765625, + "15": 0.5640078125, + "16": 0.4091171875, + "17": 0.30865625, + "18": 0.2607265625, + "19": 0.18853125, + "20": 0.18021875 + }, + "per_op_avg_response_len": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875, + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875, + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 593672, + "answer_accuracy": 0.7331745969016756, + "avg_response_len": 262.80997199059436, + "resp_tokens_sum": 212804593, + "pass_at_k": { + "pass@1": 0.7331745969016756, + "pass@2": 0.7706344784429154, + "pass@4": 0.8002934745293199, + "pass@8": 0.8244781684255242, + "pass@16": 0.8451241664967811, + "pass@32": 0.8631318988944838, + "pass@64": 0.8783587068525153, + "pass@128": 0.8912424913057224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9386292404129793, + "pass@2": 0.9558454840545374, + "pass@4": 0.9661186383781674, + "pass@8": 0.9715414375472532, + "pass@16": 0.9756175864300959, + "pass@32": 0.9796003458610456, + "pass@64": 0.9837013134257441, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977780963302753, + "pass@2": 0.9989322220616917, + "pass@4": 0.9996694776685024, + "pass@8": 0.9999677623799305, + "pass@16": 0.999999779641568, + "pass@32": 0.9999999999981484, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837158203125, + "pass@2": 0.9857337290846455, + "pass@4": 0.9882842340996436, + "pass@8": 0.9906827001964222, + "pass@16": 0.9922391181843271, + "pass@32": 0.9935574745502678, + "pass@64": 0.9951263779508654, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9695188492063492, + "pass@2": 0.976019013248344, + "pass@4": 0.9805692556287607, + "pass@8": 0.9843784633909355, + "pass@16": 0.9889107918312874, + "pass@32": 0.9940484172603424, + "pass@64": 0.9981441113782779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9948774326347305, + "pass@2": 0.9966545747088502, + "pass@4": 0.9974962398038569, + "pass@8": 0.9981117754532004, + "pass@16": 0.9988525144880754, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9937361316568047, + "pass@2": 0.9963439989749802, + "pass@4": 0.9969306777666104, + "pass@8": 0.9970380005155915, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9876302083333334, + "pass@2": 0.9916015230668282, + "pass@4": 0.9930404943372461, + "pass@8": 0.9940772665714049, + "pass@16": 0.9949131147821092, + "pass@32": 0.9957967829765609, + "pass@64": 0.9966039773874414, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9811399647887324, + "pass@2": 0.9891198569368971, + "pass@4": 0.9927404183631975, + "pass@8": 0.9952636734053902, + "pass@16": 0.9975775655877893, + "pass@32": 0.999379044270334, + "pass@64": 0.9999770279396809, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9211225212464589, + "pass@2": 0.9423424918025475, + "pass@4": 0.953929790822889, + "pass@8": 0.9614565186188683, + "pass@16": 0.9677885340638542, + "pass@32": 0.9743190927138413, + "pass@64": 0.9805943394845731, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8815986935028248, + "pass@2": 0.9165651830597449, + "pass@4": 0.934161171855637, + "pass@8": 0.9452938352373702, + "pass@16": 0.9539612829997371, + "pass@32": 0.9615647682922526, + "pass@64": 0.9668943055418103, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7734619905956113, + "pass@2": 0.8301158732999284, + "pass@4": 0.8655036610823328, + "pass@8": 0.8898255448648787, + "pass@16": 0.9097976575302069, + "pass@32": 0.92890026616942, + "pass@64": 0.9464289475637889, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.6513093847352025, + "pass@2": 0.7309117208771797, + "pass@4": 0.7865110567370666, + "pass@8": 0.8270123821269293, + "pass@16": 0.8587307099359938, + "pass@32": 0.8847289386929739, + "pass@64": 0.9048481484220781, + "pass@128": 0.9221183800623053 + }, + "15": { + "pass@1": 0.5636741329479769, + "pass@2": 0.6490562133039006, + "pass@4": 0.7107771526752794, + "pass@8": 0.7595685140977119, + "pass@16": 0.8003622070803227, + "pass@32": 0.8331029501088946, + "pass@64": 0.8575529239777137, + "pass@128": 0.8757225433526011 + }, + "16": { + "pass@1": 0.4171118951612903, + "pass@2": 0.5142795910591821, + "pass@4": 0.5949670703460455, + "pass@8": 0.6603091766742203, + "pass@16": 0.7134515965927327, + "pass@32": 0.75739935199652, + "pass@64": 0.7984302135889665, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.31419346098265893, + "pass@2": 0.3958058349642712, + "pass@4": 0.46747656868035986, + "pass@8": 0.5300846478128985, + "pass@16": 0.585240494400111, + "pass@32": 0.6313648084038579, + "pass@64": 0.669614375035597, + "pass@128": 0.6994219653179191 + }, + "18": { + "pass@1": 0.22939732142857142, + "pass@2": 0.3074957817772778, + "pass@4": 0.38013197760994144, + "pass@8": 0.44242261962567625, + "pass@16": 0.49595147536216044, + "pass@32": 0.5439121161703538, + "pass@64": 0.5859423405040576, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18125938438438438, + "pass@2": 0.24487775176751556, + "pass@4": 0.3061006349431548, + "pass@8": 0.36073586559610316, + "pass@16": 0.40994189428993016, + "pass@32": 0.45423626681010243, + "pass@64": 0.4932719107305281, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.18332394894894896, + "pass@2": 0.25119817159679353, + "pass@4": 0.3202794138345321, + "pass@8": 0.3847688536744722, + "pass@16": 0.4427514993287269, + "pass@32": 0.4952158151961868, + "pass@64": 0.5381104843576482, + "pass@128": 0.5675675675675675 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 604538, + "answer_accuracy": 0.736235872954014, + "avg_response_len": 271.6522250097428, + "resp_tokens_sum": 223059075, + "pass_at_k": { + "pass@1": 0.736235872954014, + "pass@2": 0.7735831144401895, + "pass@4": 0.8036381645152837, + "pass@8": 0.8281069190722744, + "pass@16": 0.8483726997042081, + "pass@32": 0.8655832133804009, + "pass@64": 0.8805682987802501, + "pass@128": 0.8933749025720966 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312691717791411, + "pass@2": 0.9492200708902958, + "pass@4": 0.9614929713540408, + "pass@8": 0.9706849500303524, + "pass@16": 0.9780743583738936, + "pass@32": 0.9833452492500756, + "pass@64": 0.9865077334207035, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961647727272728, + "pass@2": 0.9978696910045335, + "pass@4": 0.9990399083637271, + "pass@8": 0.9997415688663096, + "pass@16": 0.9999820837105999, + "pass@32": 0.9999999593969077, + "pass@64": 0.9999999999999987, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963480029585798, + "pass@2": 0.9970392361272888, + "pass@4": 0.9973065910215366, + "pass@8": 0.9975660568709271, + "pass@16": 0.9980247014241554, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9867753623188406, + "pass@2": 0.9882117996120052, + "pass@4": 0.9893897974709683, + "pass@8": 0.9906458350692433, + "pass@16": 0.9923677751658371, + "pass@32": 0.9946202004072046, + "pass@64": 0.996556005154711, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9901504297994269, + "pass@2": 0.9921238690973083, + "pass@4": 0.9931829475828413, + "pass@8": 0.9941730137747032, + "pass@16": 0.9951084353395996, + "pass@32": 0.9959328579290595, + "pass@64": 0.9967849648397139, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9909230491329479, + "pass@2": 0.9937133039005962, + "pass@4": 0.9946745414951744, + "pass@8": 0.9952348105100473, + "pass@16": 0.9961070653038383, + "pass@32": 0.9974785593997282, + "pass@64": 0.9991110099676848, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840008802816902, + "pass@2": 0.9890027170899413, + "pass@4": 0.9920841439714402, + "pass@8": 0.9944509310383022, + "pass@16": 0.9961687806906195, + "pass@32": 0.9970422143124665, + "pass@64": 0.9971821603034191, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.977356311000047, + "pass@4": 0.9796722011544962, + "pass@8": 0.9821828386883638, + "pass@16": 0.9851973646020666, + "pass@32": 0.988117457332326, + "pass@64": 0.9901673949748785, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.928699171686747, + "pass@2": 0.9515493074660848, + "pass@4": 0.9659594759804117, + "pass@8": 0.9766550766070073, + "pass@16": 0.9842963536346357, + "pass@32": 0.9884709338432213, + "pass@64": 0.9904158337215808, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8799960815047022, + "pass@2": 0.9035926739565081, + "pass@4": 0.9153743348695834, + "pass@8": 0.9229959569636945, + "pass@16": 0.9296964586106998, + "pass@32": 0.9365110569119893, + "pass@64": 0.94190277023684, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.7331276260504201, + "pass@2": 0.7954751290279893, + "pass@4": 0.8377238864049549, + "pass@8": 0.8699515828594602, + "pass@16": 0.8957292705806627, + "pass@32": 0.9165216166375357, + "pass@64": 0.9327622224638712, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6675804093567251, + "pass@2": 0.7419245291706962, + "pass@4": 0.796447699794104, + "pass@8": 0.8363445657414356, + "pass@16": 0.8626242889188614, + "pass@32": 0.8793508082194177, + "pass@64": 0.890769595219482, + "pass@128": 0.8976608187134503 + }, + "15": { + "pass@1": 0.5677320075757576, + "pass@2": 0.6639912908613691, + "pass@4": 0.7317696557816629, + "pass@8": 0.7795323520517282, + "pass@16": 0.8155345673543264, + "pass@32": 0.845208030617802, + "pass@64": 0.8714240604997285, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4091880341880342, + "pass@2": 0.501739616281939, + "pass@4": 0.5781482055661846, + "pass@8": 0.6406309064652197, + "pass@16": 0.6934298295552146, + "pass@32": 0.7373148005034644, + "pass@64": 0.7740434816672218, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3116022903726708, + "pass@2": 0.39830262385680026, + "pass@4": 0.47726986174476643, + "pass@8": 0.5445244698438653, + "pass@16": 0.5995497047481053, + "pass@32": 0.6445440362273804, + "pass@64": 0.6840008853131072, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.2716911764705882, + "pass@2": 0.35394171201945335, + "pass@4": 0.4276034548071196, + "pass@8": 0.4888071795579154, + "pass@16": 0.5390270190145257, + "pass@32": 0.5858273452671591, + "pass@64": 0.6344610738799759, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.17631603422619047, + "pass@2": 0.24067853334739409, + "pass@4": 0.3043591830597069, + "pass@8": 0.36333850531952205, + "pass@16": 0.41840949299425007, + "pass@32": 0.4696348856330499, + "pass@64": 0.5155865519395338, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.1794068113772455, + "pass@2": 0.24754415872978455, + "pass@4": 0.31347590112238966, + "pass@8": 0.37303300737954526, + "pass@16": 0.4267697863227694, + "pass@32": 0.4749624791026021, + "pass@64": 0.5176129128175474, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 590399, + "answer_accuracy": 0.7369375599137242, + "avg_response_len": 292.45373537106565, + "resp_tokens_sum": 234299895, + "pass_at_k": { + "pass@1": 0.7369375599137242, + "pass@2": 0.7739902768359941, + "pass@4": 0.8043156387535312, + "pass@8": 0.8298973518508928, + "pass@16": 0.8517613092704859, + "pass@32": 0.8702293942304131, + "pass@64": 0.8855494159569103, + "pass@128": 0.8985460936251798 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.9546259842519684, + "pass@4": 0.9663195143517505, + "pass@8": 0.9744687547436317, + "pass@16": 0.9804980217307362, + "pass@32": 0.9858787590967183, + "pass@64": 0.9903773541820934, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9984375, + "pass@2": 0.9992217379077615, + "pass@4": 0.9997947185173283, + "pass@8": 0.999986965598258, + "pass@16": 0.9999999656019002, + "pass@32": 0.999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970617711370262, + "pass@2": 0.9981039519294782, + "pass@4": 0.9989637462409036, + "pass@8": 0.9996508749963928, + "pass@16": 0.9999646895378788, + "pass@32": 0.9999998081458937, + "pass@64": 0.9999999999999133, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968475877192983, + "pass@2": 0.997070267532348, + "pass@4": 0.9970760217472814, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9792279411764706, + "pass@2": 0.9846865591709126, + "pass@4": 0.9878091838703983, + "pass@8": 0.9892869660212149, + "pass@16": 0.9903738755975673, + "pass@32": 0.9917880624629651, + "pass@64": 0.9933033811949977, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9955392350157729, + "pass@2": 0.9967716845922651, + "pass@4": 0.9971258053508296, + "pass@8": 0.9974048116766691, + "pass@16": 0.997893845682534, + "pass@32": 0.9986796800242242, + "pass@64": 0.9996149929208377, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9873664952531646, + "pass@2": 0.9905549966361007, + "pass@4": 0.9922436103714881, + "pass@8": 0.9935020050533728, + "pass@16": 0.9945756140766189, + "pass@32": 0.995506921497008, + "pass@64": 0.996449217370583, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9775947822822822, + "pass@2": 0.9841167446974534, + "pass@4": 0.9884578084833988, + "pass@8": 0.9913859333225822, + "pass@16": 0.9933089247757945, + "pass@32": 0.9945349372602522, + "pass@64": 0.995491156454535, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9852290996784566, + "pass@2": 0.9896970023039725, + "pass@4": 0.9907644742759243, + "pass@8": 0.9913353456864922, + "pass@16": 0.9922684000518165, + "pass@32": 0.9939805554852268, + "pass@64": 0.9967972251057042, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9187003968253968, + "pass@2": 0.9381061742282217, + "pass@4": 0.9483199814308924, + "pass@8": 0.9549746233950248, + "pass@16": 0.9607838243504727, + "pass@32": 0.9664979946638379, + "pass@64": 0.9713905391268673, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8736859709480123, + "pass@2": 0.9075217017024249, + "pass@4": 0.927535943614846, + "pass@8": 0.942757162906802, + "pass@16": 0.9558258204982006, + "pass@32": 0.9667707271291923, + "pass@64": 0.9759676186550527, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7381606867283951, + "pass@2": 0.8031985910615335, + "pass@4": 0.8491654709249306, + "pass@8": 0.884136691191738, + "pass@16": 0.9091084537688574, + "pass@32": 0.9265018813718013, + "pass@64": 0.9408495040503696, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6687917284866469, + "pass@2": 0.742496904133274, + "pass@4": 0.7915671680691242, + "pass@8": 0.8261163370467007, + "pass@16": 0.8535094321545398, + "pass@32": 0.8768087552275502, + "pass@64": 0.8966404643642354, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.560570987654321, + "pass@2": 0.6397261865704283, + "pass@4": 0.7012987609766366, + "pass@8": 0.7513312000892324, + "pass@16": 0.7905025056529893, + "pass@32": 0.8187060511329679, + "pass@64": 0.8396145819708817, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.4017330383480826, + "pass@2": 0.4958964096578634, + "pass@4": 0.5784556440953726, + "pass@8": 0.6521378744476022, + "pass@16": 0.7185738377524138, + "pass@32": 0.7734250794597738, + "pass@64": 0.8139101487404639, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.3000282379518072, + "pass@2": 0.3773594624561235, + "pass@4": 0.4456922685116164, + "pass@8": 0.5057320345187564, + "pass@16": 0.561393216655332, + "pass@32": 0.6143576371517395, + "pass@64": 0.6644245313917596, + "pass@128": 0.7108433734939759 + }, + "18": { + "pass@1": 0.2840725806451613, + "pass@2": 0.37186110934721867, + "pass@4": 0.4517761248593924, + "pass@8": 0.5177958613050213, + "pass@16": 0.5691864954935101, + "pass@32": 0.6100761487025451, + "pass@64": 0.6418813785016764, + "pass@128": 0.667741935483871 + }, + "19": { + "pass@1": 0.2082467900302115, + "pass@2": 0.27956287020957715, + "pass@4": 0.3474002794590251, + "pass@8": 0.4097443141957358, + "pass@16": 0.4660864976054333, + "pass@32": 0.5156854683282558, + "pass@64": 0.5586793575248601, + "pass@128": 0.595166163141994 + }, + "20": { + "pass@1": 0.17792792792792791, + "pass@2": 0.24336775259511484, + "pass@4": 0.3120260125142014, + "pass@8": 0.38010572901182366, + "pass@16": 0.44383337859661776, + "pass@32": 0.499497404852035, + "pass@64": 0.5444973653352457, + "pass@128": 0.5795795795795796 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fb0ebb725bf30a16d550b03e831537d37c085847 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9c4e53669dcd08bbca22961b3b968b828a9f363a5538da0bd5a76061eecf358 +size 2626413217 diff --git a/results/composition/SDAR-100M-block8-epoch1-op9-12/summary.csv b/results/composition/SDAR-100M-block8-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a8822f850af86a576410ced505225caee7f0bb43 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op9-12,0.9833871527777778,0.51230234375,0.7354477796052632,0.0,0.0,0.0,198.9335685763889,344.525071875,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch1-op9-12/summary.json b/results/composition/SDAR-100M-block8-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d97409208341e1b5a5451ef891bbdb237cd8bc01 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op9-12", + "id_acc": 0.9833871527777778, + "ood_acc": 0.51230234375, + "total_acc": 0.7354477796052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.9335685763889, + "ood_avg_resp_len": 344.525071875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 593672, + "answer_accuracy": 0.7331745969016756, + "avg_response_len": 262.80997199059436, + "resp_tokens_sum": 212804593, + "pass_at_k": { + "pass@1": 0.7331745969016756, + "pass@2": 0.7706344784429154, + "pass@4": 0.8002934745293199, + "pass@8": 0.8244781684255242, + "pass@16": 0.8451241664967811, + "pass@32": 0.8631318988944838, + "pass@64": 0.8783587068525153, + "pass@128": 0.8912424913057224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9386292404129793, + "pass@2": 0.9558454840545374, + "pass@4": 0.9661186383781674, + "pass@8": 0.9715414375472532, + "pass@16": 0.9756175864300959, + "pass@32": 0.9796003458610456, + "pass@64": 0.9837013134257441, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977780963302753, + "pass@2": 0.9989322220616917, + "pass@4": 0.9996694776685024, + "pass@8": 0.9999677623799305, + "pass@16": 0.999999779641568, + "pass@32": 0.9999999999981484, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837158203125, + "pass@2": 0.9857337290846455, + "pass@4": 0.9882842340996436, + "pass@8": 0.9906827001964222, + "pass@16": 0.9922391181843271, + "pass@32": 0.9935574745502678, + "pass@64": 0.9951263779508654, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9695188492063492, + "pass@2": 0.976019013248344, + "pass@4": 0.9805692556287607, + "pass@8": 0.9843784633909355, + "pass@16": 0.9889107918312874, + "pass@32": 0.9940484172603424, + "pass@64": 0.9981441113782779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9948774326347305, + "pass@2": 0.9966545747088502, + "pass@4": 0.9974962398038569, + "pass@8": 0.9981117754532004, + "pass@16": 0.9988525144880754, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9937361316568047, + "pass@2": 0.9963439989749802, + "pass@4": 0.9969306777666104, + "pass@8": 0.9970380005155915, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9876302083333334, + "pass@2": 0.9916015230668282, + "pass@4": 0.9930404943372461, + "pass@8": 0.9940772665714049, + "pass@16": 0.9949131147821092, + "pass@32": 0.9957967829765609, + "pass@64": 0.9966039773874414, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9811399647887324, + "pass@2": 0.9891198569368971, + "pass@4": 0.9927404183631975, + "pass@8": 0.9952636734053902, + "pass@16": 0.9975775655877893, + "pass@32": 0.999379044270334, + "pass@64": 0.9999770279396809, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9211225212464589, + "pass@2": 0.9423424918025475, + "pass@4": 0.953929790822889, + "pass@8": 0.9614565186188683, + "pass@16": 0.9677885340638542, + "pass@32": 0.9743190927138413, + "pass@64": 0.9805943394845731, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8815986935028248, + "pass@2": 0.9165651830597449, + "pass@4": 0.934161171855637, + "pass@8": 0.9452938352373702, + "pass@16": 0.9539612829997371, + "pass@32": 0.9615647682922526, + "pass@64": 0.9668943055418103, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7734619905956113, + "pass@2": 0.8301158732999284, + "pass@4": 0.8655036610823328, + "pass@8": 0.8898255448648787, + "pass@16": 0.9097976575302069, + "pass@32": 0.92890026616942, + "pass@64": 0.9464289475637889, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.6513093847352025, + "pass@2": 0.7309117208771797, + "pass@4": 0.7865110567370666, + "pass@8": 0.8270123821269293, + "pass@16": 0.8587307099359938, + "pass@32": 0.8847289386929739, + "pass@64": 0.9048481484220781, + "pass@128": 0.9221183800623053 + }, + "15": { + "pass@1": 0.5636741329479769, + "pass@2": 0.6490562133039006, + "pass@4": 0.7107771526752794, + "pass@8": 0.7595685140977119, + "pass@16": 0.8003622070803227, + "pass@32": 0.8331029501088946, + "pass@64": 0.8575529239777137, + "pass@128": 0.8757225433526011 + }, + "16": { + "pass@1": 0.4171118951612903, + "pass@2": 0.5142795910591821, + "pass@4": 0.5949670703460455, + "pass@8": 0.6603091766742203, + "pass@16": 0.7134515965927327, + "pass@32": 0.75739935199652, + "pass@64": 0.7984302135889665, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.31419346098265893, + "pass@2": 0.3958058349642712, + "pass@4": 0.46747656868035986, + "pass@8": 0.5300846478128985, + "pass@16": 0.585240494400111, + "pass@32": 0.6313648084038579, + "pass@64": 0.669614375035597, + "pass@128": 0.6994219653179191 + }, + "18": { + "pass@1": 0.22939732142857142, + "pass@2": 0.3074957817772778, + "pass@4": 0.38013197760994144, + "pass@8": 0.44242261962567625, + "pass@16": 0.49595147536216044, + "pass@32": 0.5439121161703538, + "pass@64": 0.5859423405040576, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18125938438438438, + "pass@2": 0.24487775176751556, + "pass@4": 0.3061006349431548, + "pass@8": 0.36073586559610316, + "pass@16": 0.40994189428993016, + "pass@32": 0.45423626681010243, + "pass@64": 0.4932719107305281, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.18332394894894896, + "pass@2": 0.25119817159679353, + "pass@4": 0.3202794138345321, + "pass@8": 0.3847688536744722, + "pass@16": 0.4427514993287269, + "pass@32": 0.4952158151961868, + "pass@64": 0.5381104843576482, + "pass@128": 0.5675675675675675 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 604538, + "answer_accuracy": 0.736235872954014, + "avg_response_len": 271.6522250097428, + "resp_tokens_sum": 223059075, + "pass_at_k": { + "pass@1": 0.736235872954014, + "pass@2": 0.7735831144401895, + "pass@4": 0.8036381645152837, + "pass@8": 0.8281069190722744, + "pass@16": 0.8483726997042081, + "pass@32": 0.8655832133804009, + "pass@64": 0.8805682987802501, + "pass@128": 0.8933749025720966 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312691717791411, + "pass@2": 0.9492200708902958, + "pass@4": 0.9614929713540408, + "pass@8": 0.9706849500303524, + "pass@16": 0.9780743583738936, + "pass@32": 0.9833452492500756, + "pass@64": 0.9865077334207035, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961647727272728, + "pass@2": 0.9978696910045335, + "pass@4": 0.9990399083637271, + "pass@8": 0.9997415688663096, + "pass@16": 0.9999820837105999, + "pass@32": 0.9999999593969077, + "pass@64": 0.9999999999999987, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963480029585798, + "pass@2": 0.9970392361272888, + "pass@4": 0.9973065910215366, + "pass@8": 0.9975660568709271, + "pass@16": 0.9980247014241554, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9867753623188406, + "pass@2": 0.9882117996120052, + "pass@4": 0.9893897974709683, + "pass@8": 0.9906458350692433, + "pass@16": 0.9923677751658371, + "pass@32": 0.9946202004072046, + "pass@64": 0.996556005154711, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9901504297994269, + "pass@2": 0.9921238690973083, + "pass@4": 0.9931829475828413, + "pass@8": 0.9941730137747032, + "pass@16": 0.9951084353395996, + "pass@32": 0.9959328579290595, + "pass@64": 0.9967849648397139, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9909230491329479, + "pass@2": 0.9937133039005962, + "pass@4": 0.9946745414951744, + "pass@8": 0.9952348105100473, + "pass@16": 0.9961070653038383, + "pass@32": 0.9974785593997282, + "pass@64": 0.9991110099676848, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840008802816902, + "pass@2": 0.9890027170899413, + "pass@4": 0.9920841439714402, + "pass@8": 0.9944509310383022, + "pass@16": 0.9961687806906195, + "pass@32": 0.9970422143124665, + "pass@64": 0.9971821603034191, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.977356311000047, + "pass@4": 0.9796722011544962, + "pass@8": 0.9821828386883638, + "pass@16": 0.9851973646020666, + "pass@32": 0.988117457332326, + "pass@64": 0.9901673949748785, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.928699171686747, + "pass@2": 0.9515493074660848, + "pass@4": 0.9659594759804117, + "pass@8": 0.9766550766070073, + "pass@16": 0.9842963536346357, + "pass@32": 0.9884709338432213, + "pass@64": 0.9904158337215808, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8799960815047022, + "pass@2": 0.9035926739565081, + "pass@4": 0.9153743348695834, + "pass@8": 0.9229959569636945, + "pass@16": 0.9296964586106998, + "pass@32": 0.9365110569119893, + "pass@64": 0.94190277023684, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.7331276260504201, + "pass@2": 0.7954751290279893, + "pass@4": 0.8377238864049549, + "pass@8": 0.8699515828594602, + "pass@16": 0.8957292705806627, + "pass@32": 0.9165216166375357, + "pass@64": 0.9327622224638712, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6675804093567251, + "pass@2": 0.7419245291706962, + "pass@4": 0.796447699794104, + "pass@8": 0.8363445657414356, + "pass@16": 0.8626242889188614, + "pass@32": 0.8793508082194177, + "pass@64": 0.890769595219482, + "pass@128": 0.8976608187134503 + }, + "15": { + "pass@1": 0.5677320075757576, + "pass@2": 0.6639912908613691, + "pass@4": 0.7317696557816629, + "pass@8": 0.7795323520517282, + "pass@16": 0.8155345673543264, + "pass@32": 0.845208030617802, + "pass@64": 0.8714240604997285, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4091880341880342, + "pass@2": 0.501739616281939, + "pass@4": 0.5781482055661846, + "pass@8": 0.6406309064652197, + "pass@16": 0.6934298295552146, + "pass@32": 0.7373148005034644, + "pass@64": 0.7740434816672218, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3116022903726708, + "pass@2": 0.39830262385680026, + "pass@4": 0.47726986174476643, + "pass@8": 0.5445244698438653, + "pass@16": 0.5995497047481053, + "pass@32": 0.6445440362273804, + "pass@64": 0.6840008853131072, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.2716911764705882, + "pass@2": 0.35394171201945335, + "pass@4": 0.4276034548071196, + "pass@8": 0.4888071795579154, + "pass@16": 0.5390270190145257, + "pass@32": 0.5858273452671591, + "pass@64": 0.6344610738799759, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.17631603422619047, + "pass@2": 0.24067853334739409, + "pass@4": 0.3043591830597069, + "pass@8": 0.36333850531952205, + "pass@16": 0.41840949299425007, + "pass@32": 0.4696348856330499, + "pass@64": 0.5155865519395338, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.1794068113772455, + "pass@2": 0.24754415872978455, + "pass@4": 0.31347590112238966, + "pass@8": 0.37303300737954526, + "pass@16": 0.4267697863227694, + "pass@32": 0.4749624791026021, + "pass@64": 0.5176129128175474, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 590399, + "answer_accuracy": 0.7369375599137242, + "avg_response_len": 292.45373537106565, + "resp_tokens_sum": 234299895, + "pass_at_k": { + "pass@1": 0.7369375599137242, + "pass@2": 0.7739902768359941, + "pass@4": 0.8043156387535312, + "pass@8": 0.8298973518508928, + "pass@16": 0.8517613092704859, + "pass@32": 0.8702293942304131, + "pass@64": 0.8855494159569103, + "pass@128": 0.8985460936251798 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.9546259842519684, + "pass@4": 0.9663195143517505, + "pass@8": 0.9744687547436317, + "pass@16": 0.9804980217307362, + "pass@32": 0.9858787590967183, + "pass@64": 0.9903773541820934, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9984375, + "pass@2": 0.9992217379077615, + "pass@4": 0.9997947185173283, + "pass@8": 0.999986965598258, + "pass@16": 0.9999999656019002, + "pass@32": 0.999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970617711370262, + "pass@2": 0.9981039519294782, + "pass@4": 0.9989637462409036, + "pass@8": 0.9996508749963928, + "pass@16": 0.9999646895378788, + "pass@32": 0.9999998081458937, + "pass@64": 0.9999999999999133, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968475877192983, + "pass@2": 0.997070267532348, + "pass@4": 0.9970760217472814, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9792279411764706, + "pass@2": 0.9846865591709126, + "pass@4": 0.9878091838703983, + "pass@8": 0.9892869660212149, + "pass@16": 0.9903738755975673, + "pass@32": 0.9917880624629651, + "pass@64": 0.9933033811949977, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9955392350157729, + "pass@2": 0.9967716845922651, + "pass@4": 0.9971258053508296, + "pass@8": 0.9974048116766691, + "pass@16": 0.997893845682534, + "pass@32": 0.9986796800242242, + "pass@64": 0.9996149929208377, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9873664952531646, + "pass@2": 0.9905549966361007, + "pass@4": 0.9922436103714881, + "pass@8": 0.9935020050533728, + "pass@16": 0.9945756140766189, + "pass@32": 0.995506921497008, + "pass@64": 0.996449217370583, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9775947822822822, + "pass@2": 0.9841167446974534, + "pass@4": 0.9884578084833988, + "pass@8": 0.9913859333225822, + "pass@16": 0.9933089247757945, + "pass@32": 0.9945349372602522, + "pass@64": 0.995491156454535, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9852290996784566, + "pass@2": 0.9896970023039725, + "pass@4": 0.9907644742759243, + "pass@8": 0.9913353456864922, + "pass@16": 0.9922684000518165, + "pass@32": 0.9939805554852268, + "pass@64": 0.9967972251057042, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9187003968253968, + "pass@2": 0.9381061742282217, + "pass@4": 0.9483199814308924, + "pass@8": 0.9549746233950248, + "pass@16": 0.9607838243504727, + "pass@32": 0.9664979946638379, + "pass@64": 0.9713905391268673, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8736859709480123, + "pass@2": 0.9075217017024249, + "pass@4": 0.927535943614846, + "pass@8": 0.942757162906802, + "pass@16": 0.9558258204982006, + "pass@32": 0.9667707271291923, + "pass@64": 0.9759676186550527, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7381606867283951, + "pass@2": 0.8031985910615335, + "pass@4": 0.8491654709249306, + "pass@8": 0.884136691191738, + "pass@16": 0.9091084537688574, + "pass@32": 0.9265018813718013, + "pass@64": 0.9408495040503696, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6687917284866469, + "pass@2": 0.742496904133274, + "pass@4": 0.7915671680691242, + "pass@8": 0.8261163370467007, + "pass@16": 0.8535094321545398, + "pass@32": 0.8768087552275502, + "pass@64": 0.8966404643642354, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.560570987654321, + "pass@2": 0.6397261865704283, + "pass@4": 0.7012987609766366, + "pass@8": 0.7513312000892324, + "pass@16": 0.7905025056529893, + "pass@32": 0.8187060511329679, + "pass@64": 0.8396145819708817, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.4017330383480826, + "pass@2": 0.4958964096578634, + "pass@4": 0.5784556440953726, + "pass@8": 0.6521378744476022, + "pass@16": 0.7185738377524138, + "pass@32": 0.7734250794597738, + "pass@64": 0.8139101487404639, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.3000282379518072, + "pass@2": 0.3773594624561235, + "pass@4": 0.4456922685116164, + "pass@8": 0.5057320345187564, + "pass@16": 0.561393216655332, + "pass@32": 0.6143576371517395, + "pass@64": 0.6644245313917596, + "pass@128": 0.7108433734939759 + }, + "18": { + "pass@1": 0.2840725806451613, + "pass@2": 0.37186110934721867, + "pass@4": 0.4517761248593924, + "pass@8": 0.5177958613050213, + "pass@16": 0.5691864954935101, + "pass@32": 0.6100761487025451, + "pass@64": 0.6418813785016764, + "pass@128": 0.667741935483871 + }, + "19": { + "pass@1": 0.2082467900302115, + "pass@2": 0.27956287020957715, + "pass@4": 0.3474002794590251, + "pass@8": 0.4097443141957358, + "pass@16": 0.4660864976054333, + "pass@32": 0.5156854683282558, + "pass@64": 0.5586793575248601, + "pass@128": 0.595166163141994 + }, + "20": { + "pass@1": 0.17792792792792791, + "pass@2": 0.24336775259511484, + "pass@4": 0.3120260125142014, + "pass@8": 0.38010572901182366, + "pass@16": 0.44383337859661776, + "pass@32": 0.499497404852035, + "pass@64": 0.5444973653352457, + "pass@128": 0.5795795795795796 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a6d7ab854ca2c3c59b5c32c05e56534f7449d062 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad5c1da5d374c2487724dad1b00efe1752714c57f3b2c502329489cda829fc0 +size 1789417286 diff --git a/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_metrics.json b/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fba0b03611cb67f039f4a958d83b042161a49888 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op11-14", + "id": { + "answer_accuracy": 0.9930911458333334, + "avg_response_len": 199.22368663194445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9930911458333334, + "pass@2": 0.9949381424978122, + "pass@4": 0.9961457733616635, + "pass@8": 0.9969982051319196, + "pass@16": 0.9976328783683257, + "pass@32": 0.9981484545451106, + "pass@64": 0.9986057317635817, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9525703125, + "pass@2": 0.9631371801181101, + "pass@4": 0.9709609806899138, + "pass@8": 0.9767945716645906, + "pass@16": 0.9810846355987156, + "pass@32": 0.9848159815033373, + "pass@64": 0.9885178829567154, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999609375, + "pass@2": 0.9998610974409449, + "pass@4": 0.9999817604049493, + "pass@8": 0.9999997360647032, + "pass@16": 0.9999999999758433, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99896875, + "pass@2": 0.9998102854330709, + "pass@4": 0.9999888729846269, + "pass@8": 0.9999999513947477, + "pass@16": 0.9999999999995928, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99821875, + "pass@2": 0.9987826033464566, + "pass@4": 0.9989794553805774, + "pass@8": 0.9989997726088258, + "pass@16": 0.9989999999838891, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9944609375, + "pass@2": 0.997048843503937, + "pass@4": 0.9984326085489313, + "pass@8": 0.9990485918604365, + "pass@16": 0.9993936980677535, + "pass@32": 0.9996883057343079, + "pass@64": 0.9999404409432844, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941484375, + "pass@2": 0.9958080708661419, + "pass@4": 0.9969682940569927, + "pass@8": 0.9981412225939772, + "pass@16": 0.9992175716891378, + "pass@32": 0.9998318036683483, + "pass@64": 0.9999932619722337, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9525703125, + "2": 0.9999296875, + "3": 1.0, + "4": 0.999609375, + "5": 0.99896875, + "6": 0.99821875, + "7": 0.9999140625, + "8": 0.9944609375, + "9": 0.9941484375 + }, + "per_op_avg_response_len": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376431, + "answer_accuracy": 0.9932006712259371, + "avg_response_len": 189.76862757514354, + "resp_tokens_sum": 71923828, + "pass_at_k": { + "pass@1": 0.9932006712259371, + "pass@2": 0.995337226796119, + "pass@4": 0.9966168958806693, + "pass@8": 0.9974370308941563, + "pass@16": 0.9980251463797202, + "pass@32": 0.9985125017885638, + "pass@64": 0.9989902516880416, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9550377949852508, + "pass@2": 0.9653236272733609, + "pass@4": 0.9726180897741764, + "pass@8": 0.9783762148389291, + "pass@16": 0.9829116146743994, + "pass@32": 0.9870127565970104, + "pass@64": 0.991180340449138, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997809579439252, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998779296875, + "pass@2": 0.9995659295029528, + "pass@4": 0.9999430012654666, + "pass@8": 0.9999991752021973, + "pass@16": 0.9999999999245107, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982638888888888, + "pass@2": 0.9995289651293588, + "pass@4": 0.9999661601823582, + "pass@8": 0.9999998458721072, + "pass@16": 0.9999999999987073, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989006362275449, + "pass@2": 0.9996018052477722, + "pass@4": 0.9999499412311484, + "pass@8": 0.9999993414788002, + "pass@16": 0.9999999999517831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999075443786982, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9933143028846154, + "pass@2": 0.997447096456693, + "pass@4": 0.999438021268976, + "pass@8": 0.9999627194608325, + "pass@16": 0.9999998418611676, + "pass@32": 0.9999999999993773, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9976620550072086, + "pass@4": 0.9985522935513342, + "pass@8": 0.9993061284706786, + "pass@16": 0.9998463955935609, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387539, + "answer_accuracy": 0.9920211132044561, + "avg_response_len": 197.2330285468545, + "resp_tokens_sum": 77050266, + "pass_at_k": { + "pass@1": 0.9920211132044561, + "pass@2": 0.9936430367720664, + "pass@4": 0.9947641457272788, + "pass@8": 0.9956322126604567, + "pass@16": 0.9963545198515974, + "pass@32": 0.9970399392150072, + "pass@64": 0.9977077449554393, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9402319785276073, + "pass@2": 0.952008281604754, + "pass@4": 0.9611208099562711, + "pass@8": 0.967649685144444, + "pass@16": 0.9723239331864337, + "pass@32": 0.976739705975646, + "pass@64": 0.9818104174845861, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999411231884058, + "pass@2": 0.9999315302978432, + "pass@4": 0.9999989373610907, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966869627507163, + "pass@2": 0.9970676905895359, + "pass@4": 0.9971333691633818, + "pass@8": 0.9971346702346441, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9998871026011561, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965889084507042, + "pass@2": 0.9972960796273704, + "pass@4": 0.9975222988851745, + "pass@8": 0.9978309304389954, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932400823353293, + "pass@2": 0.99476379426187, + "pass@4": 0.995734111922387, + "pass@8": 0.9969632799932823, + "pass@16": 0.9984388741597237, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950416, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380071, + "answer_accuracy": 0.9940758913625711, + "avg_response_len": 210.630421932541, + "resp_tokens_sum": 80531593, + "pass_at_k": { + "pass@1": 0.9940758913625711, + "pass@2": 0.995865820458206, + "pass@4": 0.9970904448586044, + "pass@8": 0.9979589168630615, + "pass@16": 0.9985502016396067, + "pass@32": 0.9989202138017588, + "pass@64": 0.9991420867157488, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.962080223880597, + "pass@2": 0.9717545393113174, + "pass@4": 0.9788598931663394, + "pass@8": 0.984093246200315, + "pass@16": 0.9877611821054247, + "pass@32": 0.9904522772205684, + "pass@64": 0.9923508998342762, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999172794117647, + "pass@2": 0.9999478925428441, + "pass@4": 0.9999997033459054, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991867113564669, + "pass@2": 0.9998074964604189, + "pass@4": 0.9999893664711469, + "pass@8": 0.9999999767918476, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9932667042042042, + "pass@2": 0.9964121355607576, + "pass@4": 0.9984610504767983, + "pass@8": 0.999490219467908, + "pass@16": 0.9999277447691266, + "pass@32": 0.999999100781228, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992941117363344, + "pass@2": 0.9948132927817303, + "pass@4": 0.9964856477948294, + "pass@8": 0.9980765642094214, + "pass@16": 0.999336068437543, + "pass@32": 0.9999146790894352, + "pass@64": 0.9999995549101495, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.67023828125, + "avg_response_len": 370.87193515625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.67023828125, + "pass@2": 0.7391869832677044, + "pass@4": 0.7914964344206904, + "pass@8": 0.8305876741468599, + "pass@16": 0.8598251061006209, + "pass@32": 0.8822750532094229, + "pass@64": 0.9001723511612838, + "pass@128": 0.9149 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9613515625, + "pass@2": 0.9716198326771655, + "pass@4": 0.9771861735095614, + "pass@8": 0.9808070057015184, + "pass@16": 0.9837384231215315, + "pass@32": 0.9858625291421717, + "pass@64": 0.9869666845018911, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9076796875, + "pass@2": 0.9273491633858271, + "pass@4": 0.9397097274090743, + "pass@8": 0.9481943683164651, + "pass@16": 0.9539304509846932, + "pass@32": 0.9579559040128569, + "pass@64": 0.9609342516531564, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.849375, + "pass@2": 0.8836884842519688, + "pass@4": 0.9050689845331829, + "pass@8": 0.919391571172967, + "pass@16": 0.930207215681193, + "pass@32": 0.939676797003826, + "pass@64": 0.9483149333749666, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.843671875, + "pass@2": 0.8859703494094502, + "pass@4": 0.9096745502437197, + "pass@8": 0.9264951601373707, + "pass@16": 0.9398242906228735, + "pass@32": 0.9497296044610397, + "pass@64": 0.9577593392189998, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8058515625, + "pass@2": 0.8670836614173242, + "pass@4": 0.9038101552305964, + "pass@8": 0.9279624319017417, + "pass@16": 0.9447318076178691, + "pass@32": 0.9566893837117887, + "pass@64": 0.9647829004058682, + "pass@128": 0.971 + }, + "16": { + "pass@1": 0.64621875, + "pass@2": 0.7415820620078745, + "pass@4": 0.8050657452193474, + "pass@8": 0.8463069193831734, + "pass@16": 0.8743385001840214, + "pass@32": 0.8966440530869019, + "pass@64": 0.9157644672574666, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.56553125, + "pass@2": 0.6692714074803152, + "pass@4": 0.7414758831083603, + "pass@8": 0.7884725208544237, + "pass@16": 0.8217109948566683, + "pass@32": 0.8488122736046942, + "pass@64": 0.8727472203835355, + "pass@128": 0.894 + }, + "18": { + "pass@1": 0.4961796875, + "pass@2": 0.6022688238188982, + "pass@4": 0.6869041670416197, + "pass@8": 0.7508196935626941, + "pass@16": 0.7979812625700984, + "pass@32": 0.8342351494830981, + "pass@64": 0.8626833370727636, + "pass@128": 0.884 + }, + "19": { + "pass@1": 0.3210546875, + "pass@2": 0.4304410679133851, + "pass@4": 0.5329773877952755, + "pass@8": 0.6194907254128081, + "pass@16": 0.686649728649235, + "pass@32": 0.7366339679870643, + "pass@64": 0.7755140138496416, + "pass@128": 0.808 + }, + "20": { + "pass@1": 0.30546875, + "pass@2": 0.4125949803149601, + "pass@4": 0.5130915701162351, + "pass@8": 0.597936345025493, + "pass@16": 0.6651383867179285, + "pass@32": 0.7165108696007512, + "pass@64": 0.7562563638944912, + "pass@128": 0.79 + } + }, + "per_op_accuracy": { + "11": 0.9613515625, + "12": 0.9076796875, + "13": 0.849375, + "14": 0.843671875, + "15": 0.8058515625, + "16": 0.64621875, + "17": 0.56553125, + "18": 0.4961796875, + "19": 0.3210546875, + "20": 0.30546875 + }, + "per_op_avg_response_len": { + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 292022, + "answer_accuracy": 0.6779856983655275, + "avg_response_len": 353.17834323922733, + "resp_tokens_sum": 152120976, + "pass_at_k": { + "pass@1": 0.6779856983655275, + "pass@2": 0.7444212437551985, + "pass@4": 0.7953746970986009, + "pass@8": 0.8348435843436636, + "pass@16": 0.8648672389343015, + "pass@32": 0.8874072691956361, + "pass@64": 0.9046577103739157, + "pass@128": 0.9188707280832095 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9609375, + "pass@2": 0.9711451060649998, + "pass@4": 0.9770741222857058, + "pass@8": 0.9812611025364644, + "pass@16": 0.9842944002686944, + "pass@32": 0.9861828830088263, + "pass@64": 0.9872429473453521, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.9021671963276836, + "pass@2": 0.9215281484941499, + "pass@4": 0.9347171227537232, + "pass@8": 0.9446575312767541, + "pass@16": 0.9525455705033121, + "pass@32": 0.9580749193035334, + "pass@64": 0.9613374661256407, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8692202194357367, + "pass@2": 0.9024653352010464, + "pass@4": 0.9221733035721631, + "pass@8": 0.9359308821175786, + "pass@16": 0.9484843594916188, + "pass@32": 0.960115260632947, + "pass@64": 0.9696726490165624, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8432389018691588, + "pass@2": 0.8816467669929107, + "pass@4": 0.9054283430692655, + "pass@8": 0.9252522133459742, + "pass@16": 0.9410032100908022, + "pass@32": 0.9510890172203004, + "pass@64": 0.9586071337072141, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.8159320809248555, + "pass@2": 0.8758590869782895, + "pass@4": 0.9136971456963835, + "pass@8": 0.9391793646317208, + "pass@16": 0.9568471423736444, + "pass@32": 0.9685935199216499, + "pass@64": 0.9754437045824256, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6567792338709677, + "pass@2": 0.740235664846329, + "pass@4": 0.795893839278154, + "pass@8": 0.8355960749083245, + "pass@16": 0.864558190334898, + "pass@32": 0.8889121743620298, + "pass@64": 0.9101876710408604, + "pass@128": 0.9258064516129032 + }, + "17": { + "pass@1": 0.5896405346820809, + "pass@2": 0.689234175162714, + "pass@4": 0.7562385357538394, + "pass@8": 0.7996128076425166, + "pass@16": 0.8318120005184033, + "pass@32": 0.8596979457171261, + "pass@64": 0.8843409721762664, + "pass@128": 0.9046242774566474 + }, + "18": { + "pass@1": 0.47859375, + "pass@2": 0.582545345894263, + "pass@4": 0.665977828753548, + "pass@8": 0.731456928764178, + "pass@16": 0.780729199135965, + "pass@32": 0.8185404669626503, + "pass@64": 0.848842739464603, + "pass@128": 0.8742857142857143 + }, + "19": { + "pass@1": 0.32244744744744747, + "pass@2": 0.42991527452649503, + "pass@4": 0.5297148988133239, + "pass@8": 0.6158184040059627, + "pass@16": 0.6829310379104327, + "pass@32": 0.7314577265217167, + "pass@64": 0.7668547527202739, + "pass@128": 0.7927927927927928 + }, + "20": { + "pass@1": 0.33054147897897895, + "pass@2": 0.441439224657729, + "pass@4": 0.5459787081794953, + "pass@8": 0.6340915487675354, + "pass@16": 0.7009979125456074, + "pass@32": 0.7479033979165912, + "pass@64": 0.781451169312069, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 291271, + "answer_accuracy": 0.6766442722271782, + "avg_response_len": 366.3032239629795, + "resp_tokens_sum": 157680351, + "pass_at_k": { + "pass@1": 0.6766442722271782, + "pass@2": 0.7441928255845854, + "pass@4": 0.7948994997112895, + "pass@8": 0.8326424323324724, + "pass@16": 0.8614864198568543, + "pass@32": 0.884597208281839, + "pass@64": 0.903587135802124, + "pass@128": 0.9191198334820101 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9623258659638554, + "pass@2": 0.9707566733469312, + "pass@4": 0.9751945817578527, + "pass@8": 0.978528567411368, + "pass@16": 0.9816759959049333, + "pass@32": 0.9840488998900503, + "pass@64": 0.9848917427571217, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9100215517241379, + "pass@2": 0.9266520160195493, + "pass@4": 0.9344253890285655, + "pass@8": 0.9383279633369872, + "pass@16": 0.9410428618385669, + "pass@32": 0.9436759703758117, + "pass@64": 0.9467110995185979, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.8494616596638656, + "pass@2": 0.8807649043869519, + "pass@4": 0.900884443803768, + "pass@8": 0.9155814360333341, + "pass@16": 0.9267145595077136, + "pass@32": 0.9359428582481637, + "pass@64": 0.9445210713800876, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.8588267543859649, + "pass@2": 0.9018694312059676, + "pass@4": 0.9242850279679948, + "pass@8": 0.9383757151048715, + "pass@16": 0.9484554246399113, + "pass@32": 0.9561951272112444, + "pass@64": 0.9627628804004219, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.8121685606060606, + "pass@2": 0.8770117513719882, + "pass@4": 0.915564349342695, + "pass@8": 0.9404922694298705, + "pass@16": 0.9573654432962957, + "pass@32": 0.9699346349116533, + "pass@64": 0.9789174437718314, + "pass@128": 0.9848484848484849 + }, + "16": { + "pass@1": 0.6486823361823362, + "pass@2": 0.7438652500168247, + "pass@4": 0.805919283326763, + "pass@8": 0.8449027197071314, + "pass@16": 0.8727992336539356, + "pass@32": 0.8965191234491018, + "pass@64": 0.9150418318628365, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.5890916149068323, + "pass@2": 0.6926184159045329, + "pass@4": 0.764558463286033, + "pass@8": 0.8095983074474769, + "pass@16": 0.8386906126837259, + "pass@32": 0.8617211263710541, + "pass@64": 0.8851656158646087, + "pass@128": 0.9099378881987578 + }, + "18": { + "pass@1": 0.5270220588235294, + "pass@2": 0.6297294754515979, + "pass@4": 0.7110232550894364, + "pass@8": 0.7729100372599288, + "pass@16": 0.8200711561466233, + "pass@32": 0.8564476725925791, + "pass@64": 0.8842089254988268, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.31570870535714285, + "pass@2": 0.4234084997187852, + "pass@4": 0.5270488630327456, + "pass@8": 0.6161440847378656, + "pass@16": 0.6883030755974877, + "pass@32": 0.7454121901545792, + "pass@64": 0.791591256811504, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.29381081586826346, + "pass@2": 0.39535287201188124, + "pass@4": 0.48994442414383815, + "pass@8": 0.5711483363826257, + "pass@16": 0.6389980310343075, + "pass@32": 0.6950049674288274, + "pass@64": 0.740922892472733, + "pass@128": 0.781437125748503 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 274612, + "answer_accuracy": 0.6556865067237164, + "avg_response_len": 393.7642067160758, + "resp_tokens_sum": 164914750, + "pass_at_k": { + "pass@1": 0.6556865067237164, + "pass@2": 0.7286588860024493, + "pass@4": 0.7840102295052386, + "pass@8": 0.8240988937097103, + "pass@16": 0.852932143035952, + "pass@32": 0.8746102260999513, + "pass@64": 0.8920497489309196, + "pass@128": 0.906479217603912 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9607886904761904, + "pass@2": 0.9730615704286967, + "pass@4": 0.9794108165050797, + "pass@8": 0.9826995305573731, + "pass@16": 0.9852891085277606, + "pass@32": 0.9874150370684425, + "pass@64": 0.9888440174400547, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9113627675840978, + "pass@2": 0.934330904307833, + "pass@4": 0.950269623407166, + "pass@8": 0.9616482628134414, + "pass@16": 0.9680019758410323, + "pass@32": 0.9717576392340137, + "pass@64": 0.9743729293524362, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8297405478395061, + "pass@2": 0.8684227514095461, + "pass@4": 0.8928393649867837, + "pass@8": 0.9073057009677743, + "pass@16": 0.916060516231829, + "pass@32": 0.9236679890966697, + "pass@64": 0.9314670859752837, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.82870456231454, + "pass@2": 0.87395368466553, + "pass@4": 0.898891906686738, + "pass@8": 0.9156222702891602, + "pass@16": 0.9299421512073494, + "pass@32": 0.9418732831664041, + "pass@64": 0.9518740184036798, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.7886525848765432, + "pass@2": 0.8476004301545641, + "pass@4": 0.8812799615325858, + "pass@8": 0.903221984034905, + "pass@16": 0.9189262347802786, + "pass@32": 0.9304864083273199, + "pass@64": 0.9390019202952004, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.634010877581121, + "pass@2": 0.740449271827747, + "pass@4": 0.8125692819924054, + "pass@8": 0.8575554027858113, + "pass@16": 0.8848759001996248, + "pass@32": 0.9038438545841817, + "pass@64": 0.9216124078198928, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.517554593373494, + "pass@2": 0.6258230510625178, + "pass@4": 0.7037033872271984, + "pass@8": 0.756372995216946, + "pass@16": 0.794715859617898, + "pass@32": 0.8249475352261741, + "pass@64": 0.8486202280185033, + "pass@128": 0.8674698795180723 + }, + "18": { + "pass@1": 0.48220766129032255, + "pass@2": 0.5944191325882653, + "pass@4": 0.6840774846692548, + "pass@8": 0.7484527607317959, + "pass@16": 0.7932317734924506, + "pass@32": 0.8275928302086873, + "pass@64": 0.8547011083566157, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.325080249244713, + "pass@2": 0.43810883864214883, + "pass@4": 0.5422776692188405, + "pass@8": 0.6265824302323246, + "pass@16": 0.6887125668408031, + "pass@32": 0.7329306621250592, + "pass@64": 0.767905495181644, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.29208896396396394, + "pass@2": 0.401044622378284, + "pass@4": 0.5034210889742385, + "pass@8": 0.5886495943967197, + "pass@16": 0.6554977161404892, + "pass@32": 0.7066888257456404, + "pass@64": 0.746441076359396, + "pass@128": 0.7777777777777778 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8231685855263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8231685855263158, + "pass@2": 0.8603322692188005, + "pass@4": 0.8884355949716634, + "pass@8": 0.9094137151397931, + "pass@16": 0.9251024719116253, + "pass@32": 0.9371624538421249, + "pass@64": 0.946798689341318, + "pass@128": 0.9547368421052631 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9525703125, + "pass@2": 0.9631371801181101, + "pass@4": 0.9709609806899138, + "pass@8": 0.9767945716645906, + "pass@16": 0.9810846355987156, + "pass@32": 0.9848159815033373, + "pass@64": 0.9885178829567154, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999609375, + "pass@2": 0.9998610974409449, + "pass@4": 0.9999817604049493, + "pass@8": 0.9999997360647032, + "pass@16": 0.9999999999758433, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99896875, + "pass@2": 0.9998102854330709, + "pass@4": 0.9999888729846269, + "pass@8": 0.9999999513947477, + "pass@16": 0.9999999999995928, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99821875, + "pass@2": 0.9987826033464566, + "pass@4": 0.9989794553805774, + "pass@8": 0.9989997726088258, + "pass@16": 0.9989999999838891, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9944609375, + "pass@2": 0.997048843503937, + "pass@4": 0.9984326085489313, + "pass@8": 0.9990485918604365, + "pass@16": 0.9993936980677535, + "pass@32": 0.9996883057343079, + "pass@64": 0.9999404409432844, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941484375, + "pass@2": 0.9958080708661419, + "pass@4": 0.9969682940569927, + "pass@8": 0.9981412225939772, + "pass@16": 0.9992175716891378, + "pass@32": 0.9998318036683483, + "pass@64": 0.9999932619722337, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9613515625, + "pass@2": 0.9716198326771655, + "pass@4": 0.9771861735095614, + "pass@8": 0.9808070057015184, + "pass@16": 0.9837384231215315, + "pass@32": 0.9858625291421717, + "pass@64": 0.9869666845018911, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9076796875, + "pass@2": 0.9273491633858271, + "pass@4": 0.9397097274090743, + "pass@8": 0.9481943683164651, + "pass@16": 0.9539304509846932, + "pass@32": 0.9579559040128569, + "pass@64": 0.9609342516531564, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.849375, + "pass@2": 0.8836884842519688, + "pass@4": 0.9050689845331829, + "pass@8": 0.919391571172967, + "pass@16": 0.930207215681193, + "pass@32": 0.939676797003826, + "pass@64": 0.9483149333749666, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.843671875, + "pass@2": 0.8859703494094502, + "pass@4": 0.9096745502437197, + "pass@8": 0.9264951601373707, + "pass@16": 0.9398242906228735, + "pass@32": 0.9497296044610397, + "pass@64": 0.9577593392189998, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8058515625, + "pass@2": 0.8670836614173242, + "pass@4": 0.9038101552305964, + "pass@8": 0.9279624319017417, + "pass@16": 0.9447318076178691, + "pass@32": 0.9566893837117887, + "pass@64": 0.9647829004058682, + "pass@128": 0.971 + }, + "16": { + "pass@1": 0.64621875, + "pass@2": 0.7415820620078745, + "pass@4": 0.8050657452193474, + "pass@8": 0.8463069193831734, + "pass@16": 0.8743385001840214, + "pass@32": 0.8966440530869019, + "pass@64": 0.9157644672574666, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.56553125, + "pass@2": 0.6692714074803152, + "pass@4": 0.7414758831083603, + "pass@8": 0.7884725208544237, + "pass@16": 0.8217109948566683, + "pass@32": 0.8488122736046942, + "pass@64": 0.8727472203835355, + "pass@128": 0.894 + }, + "18": { + "pass@1": 0.4961796875, + "pass@2": 0.6022688238188982, + "pass@4": 0.6869041670416197, + "pass@8": 0.7508196935626941, + "pass@16": 0.7979812625700984, + "pass@32": 0.8342351494830981, + "pass@64": 0.8626833370727636, + "pass@128": 0.884 + }, + "19": { + "pass@1": 0.3210546875, + "pass@2": 0.4304410679133851, + "pass@4": 0.5329773877952755, + "pass@8": 0.6194907254128081, + "pass@16": 0.686649728649235, + "pass@32": 0.7366339679870643, + "pass@64": 0.7755140138496416, + "pass@128": 0.808 + }, + "20": { + "pass@1": 0.30546875, + "pass@2": 0.4125949803149601, + "pass@4": 0.5130915701162351, + "pass@8": 0.597936345025493, + "pass@16": 0.6651383867179285, + "pass@32": 0.7165108696007512, + "pass@64": 0.7562563638944912, + "pass@128": 0.79 + } + }, + "per_op_accuracy": { + "10": 0.9525703125, + "2": 0.9999296875, + "3": 1.0, + "4": 0.999609375, + "5": 0.99896875, + "6": 0.99821875, + "7": 0.9999140625, + "8": 0.9944609375, + "9": 0.9941484375, + "11": 0.9613515625, + "12": 0.9076796875, + "13": 0.849375, + "14": 0.843671875, + "15": 0.8058515625, + "16": 0.64621875, + "17": 0.56553125, + "18": 0.4961796875, + "19": 0.3210546875, + "20": 0.30546875 + }, + "per_op_avg_response_len": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234, + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234, + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 668453, + "answer_accuracy": 0.8255278315681315, + "avg_response_len": 276.6914371245653, + "resp_tokens_sum": 224044804, + "pass_at_k": { + "pass@1": 0.8255278315681315, + "pass@2": 0.861867058770073, + "pass@4": 0.8895697888775553, + "pass@8": 0.9109484207704667, + "pass@16": 0.9271942329187942, + "pass@32": 0.93941210538085, + "pass@64": 0.9488117816403033, + "pass@128": 0.9565286120771419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9550377949852508, + "pass@2": 0.9653236272733609, + "pass@4": 0.9726180897741764, + "pass@8": 0.9783762148389291, + "pass@16": 0.9829116146743994, + "pass@32": 0.9870127565970104, + "pass@64": 0.991180340449138, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997809579439252, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998779296875, + "pass@2": 0.9995659295029528, + "pass@4": 0.9999430012654666, + "pass@8": 0.9999991752021973, + "pass@16": 0.9999999999245107, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982638888888888, + "pass@2": 0.9995289651293588, + "pass@4": 0.9999661601823582, + "pass@8": 0.9999998458721072, + "pass@16": 0.9999999999987073, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989006362275449, + "pass@2": 0.9996018052477722, + "pass@4": 0.9999499412311484, + "pass@8": 0.9999993414788002, + "pass@16": 0.9999999999517831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999075443786982, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9933143028846154, + "pass@2": 0.997447096456693, + "pass@4": 0.999438021268976, + "pass@8": 0.9999627194608325, + "pass@16": 0.9999998418611676, + "pass@32": 0.9999999999993773, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9976620550072086, + "pass@4": 0.9985522935513342, + "pass@8": 0.9993061284706786, + "pass@16": 0.9998463955935609, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9609375, + "pass@2": 0.9711451060649998, + "pass@4": 0.9770741222857058, + "pass@8": 0.9812611025364644, + "pass@16": 0.9842944002686944, + "pass@32": 0.9861828830088263, + "pass@64": 0.9872429473453521, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.9021671963276836, + "pass@2": 0.9215281484941499, + "pass@4": 0.9347171227537232, + "pass@8": 0.9446575312767541, + "pass@16": 0.9525455705033121, + "pass@32": 0.9580749193035334, + "pass@64": 0.9613374661256407, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8692202194357367, + "pass@2": 0.9024653352010464, + "pass@4": 0.9221733035721631, + "pass@8": 0.9359308821175786, + "pass@16": 0.9484843594916188, + "pass@32": 0.960115260632947, + "pass@64": 0.9696726490165624, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8432389018691588, + "pass@2": 0.8816467669929107, + "pass@4": 0.9054283430692655, + "pass@8": 0.9252522133459742, + "pass@16": 0.9410032100908022, + "pass@32": 0.9510890172203004, + "pass@64": 0.9586071337072141, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.8159320809248555, + "pass@2": 0.8758590869782895, + "pass@4": 0.9136971456963835, + "pass@8": 0.9391793646317208, + "pass@16": 0.9568471423736444, + "pass@32": 0.9685935199216499, + "pass@64": 0.9754437045824256, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6567792338709677, + "pass@2": 0.740235664846329, + "pass@4": 0.795893839278154, + "pass@8": 0.8355960749083245, + "pass@16": 0.864558190334898, + "pass@32": 0.8889121743620298, + "pass@64": 0.9101876710408604, + "pass@128": 0.9258064516129032 + }, + "17": { + "pass@1": 0.5896405346820809, + "pass@2": 0.689234175162714, + "pass@4": 0.7562385357538394, + "pass@8": 0.7996128076425166, + "pass@16": 0.8318120005184033, + "pass@32": 0.8596979457171261, + "pass@64": 0.8843409721762664, + "pass@128": 0.9046242774566474 + }, + "18": { + "pass@1": 0.47859375, + "pass@2": 0.582545345894263, + "pass@4": 0.665977828753548, + "pass@8": 0.731456928764178, + "pass@16": 0.780729199135965, + "pass@32": 0.8185404669626503, + "pass@64": 0.848842739464603, + "pass@128": 0.8742857142857143 + }, + "19": { + "pass@1": 0.32244744744744747, + "pass@2": 0.42991527452649503, + "pass@4": 0.5297148988133239, + "pass@8": 0.6158184040059627, + "pass@16": 0.6829310379104327, + "pass@32": 0.7314577265217167, + "pass@64": 0.7668547527202739, + "pass@128": 0.7927927927927928 + }, + "20": { + "pass@1": 0.33054147897897895, + "pass@2": 0.441439224657729, + "pass@4": 0.5459787081794953, + "pass@8": 0.6340915487675354, + "pass@16": 0.7009979125456074, + "pass@32": 0.7479033979165912, + "pass@64": 0.781451169312069, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 678810, + "answer_accuracy": 0.8266879384255651, + "avg_response_len": 285.86639833398283, + "resp_tokens_sum": 234730617, + "pass_at_k": { + "pass@1": 0.8266879384255651, + "pass@2": 0.8628712425049453, + "pass@4": 0.8899870912375201, + "pass@8": 0.9101864400582642, + "pass@16": 0.9256512587008077, + "pass@32": 0.9380929549393652, + "pass@64": 0.9483659509441283, + "pass@128": 0.9568199532346064 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9402319785276073, + "pass@2": 0.952008281604754, + "pass@4": 0.9611208099562711, + "pass@8": 0.967649685144444, + "pass@16": 0.9723239331864337, + "pass@32": 0.976739705975646, + "pass@64": 0.9818104174845861, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999411231884058, + "pass@2": 0.9999315302978432, + "pass@4": 0.9999989373610907, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966869627507163, + "pass@2": 0.9970676905895359, + "pass@4": 0.9971333691633818, + "pass@8": 0.9971346702346441, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9998871026011561, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965889084507042, + "pass@2": 0.9972960796273704, + "pass@4": 0.9975222988851745, + "pass@8": 0.9978309304389954, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932400823353293, + "pass@2": 0.99476379426187, + "pass@4": 0.995734111922387, + "pass@8": 0.9969632799932823, + "pass@16": 0.9984388741597237, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950416, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9623258659638554, + "pass@2": 0.9707566733469312, + "pass@4": 0.9751945817578527, + "pass@8": 0.978528567411368, + "pass@16": 0.9816759959049333, + "pass@32": 0.9840488998900503, + "pass@64": 0.9848917427571217, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9100215517241379, + "pass@2": 0.9266520160195493, + "pass@4": 0.9344253890285655, + "pass@8": 0.9383279633369872, + "pass@16": 0.9410428618385669, + "pass@32": 0.9436759703758117, + "pass@64": 0.9467110995185979, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.8494616596638656, + "pass@2": 0.8807649043869519, + "pass@4": 0.900884443803768, + "pass@8": 0.9155814360333341, + "pass@16": 0.9267145595077136, + "pass@32": 0.9359428582481637, + "pass@64": 0.9445210713800876, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.8588267543859649, + "pass@2": 0.9018694312059676, + "pass@4": 0.9242850279679948, + "pass@8": 0.9383757151048715, + "pass@16": 0.9484554246399113, + "pass@32": 0.9561951272112444, + "pass@64": 0.9627628804004219, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.8121685606060606, + "pass@2": 0.8770117513719882, + "pass@4": 0.915564349342695, + "pass@8": 0.9404922694298705, + "pass@16": 0.9573654432962957, + "pass@32": 0.9699346349116533, + "pass@64": 0.9789174437718314, + "pass@128": 0.9848484848484849 + }, + "16": { + "pass@1": 0.6486823361823362, + "pass@2": 0.7438652500168247, + "pass@4": 0.805919283326763, + "pass@8": 0.8449027197071314, + "pass@16": 0.8727992336539356, + "pass@32": 0.8965191234491018, + "pass@64": 0.9150418318628365, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.5890916149068323, + "pass@2": 0.6926184159045329, + "pass@4": 0.764558463286033, + "pass@8": 0.8095983074474769, + "pass@16": 0.8386906126837259, + "pass@32": 0.8617211263710541, + "pass@64": 0.8851656158646087, + "pass@128": 0.9099378881987578 + }, + "18": { + "pass@1": 0.5270220588235294, + "pass@2": 0.6297294754515979, + "pass@4": 0.7110232550894364, + "pass@8": 0.7729100372599288, + "pass@16": 0.8200711561466233, + "pass@32": 0.8564476725925791, + "pass@64": 0.8842089254988268, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.31570870535714285, + "pass@2": 0.4234084997187852, + "pass@4": 0.5270488630327456, + "pass@8": 0.6161440847378656, + "pass@16": 0.6883030755974877, + "pass@32": 0.7454121901545792, + "pass@64": 0.791591256811504, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.29381081586826346, + "pass@2": 0.39535287201188124, + "pass@4": 0.48994442414383815, + "pass@8": 0.5711483363826257, + "pass@16": 0.6389980310343075, + "pass@32": 0.6950049674288274, + "pass@64": 0.740922892472733, + "pass@128": 0.781437125748503 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 654683, + "answer_accuracy": 0.8171770150982585, + "avg_response_len": 306.3667606147148, + "resp_tokens_sum": 245446343, + "pass_at_k": { + "pass@1": 0.8171770150982585, + "pass@2": 0.8561787954479328, + "pass@4": 0.8856990940619518, + "pass@8": 0.9070705967228174, + "pass@16": 0.9224258546590777, + "pass@32": 0.9339350277080865, + "pass@64": 0.9431577235216402, + "pass@128": 0.9507908611599297 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.962080223880597, + "pass@2": 0.9717545393113174, + "pass@4": 0.9788598931663394, + "pass@8": 0.984093246200315, + "pass@16": 0.9877611821054247, + "pass@32": 0.9904522772205684, + "pass@64": 0.9923508998342762, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999172794117647, + "pass@2": 0.9999478925428441, + "pass@4": 0.9999997033459054, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991867113564669, + "pass@2": 0.9998074964604189, + "pass@4": 0.9999893664711469, + "pass@8": 0.9999999767918476, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9932667042042042, + "pass@2": 0.9964121355607576, + "pass@4": 0.9984610504767983, + "pass@8": 0.999490219467908, + "pass@16": 0.9999277447691266, + "pass@32": 0.999999100781228, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992941117363344, + "pass@2": 0.9948132927817303, + "pass@4": 0.9964856477948294, + "pass@8": 0.9980765642094214, + "pass@16": 0.999336068437543, + "pass@32": 0.9999146790894352, + "pass@64": 0.9999995549101495, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9607886904761904, + "pass@2": 0.9730615704286967, + "pass@4": 0.9794108165050797, + "pass@8": 0.9826995305573731, + "pass@16": 0.9852891085277606, + "pass@32": 0.9874150370684425, + "pass@64": 0.9888440174400547, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9113627675840978, + "pass@2": 0.934330904307833, + "pass@4": 0.950269623407166, + "pass@8": 0.9616482628134414, + "pass@16": 0.9680019758410323, + "pass@32": 0.9717576392340137, + "pass@64": 0.9743729293524362, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8297405478395061, + "pass@2": 0.8684227514095461, + "pass@4": 0.8928393649867837, + "pass@8": 0.9073057009677743, + "pass@16": 0.916060516231829, + "pass@32": 0.9236679890966697, + "pass@64": 0.9314670859752837, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.82870456231454, + "pass@2": 0.87395368466553, + "pass@4": 0.898891906686738, + "pass@8": 0.9156222702891602, + "pass@16": 0.9299421512073494, + "pass@32": 0.9418732831664041, + "pass@64": 0.9518740184036798, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.7886525848765432, + "pass@2": 0.8476004301545641, + "pass@4": 0.8812799615325858, + "pass@8": 0.903221984034905, + "pass@16": 0.9189262347802786, + "pass@32": 0.9304864083273199, + "pass@64": 0.9390019202952004, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.634010877581121, + "pass@2": 0.740449271827747, + "pass@4": 0.8125692819924054, + "pass@8": 0.8575554027858113, + "pass@16": 0.8848759001996248, + "pass@32": 0.9038438545841817, + "pass@64": 0.9216124078198928, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.517554593373494, + "pass@2": 0.6258230510625178, + "pass@4": 0.7037033872271984, + "pass@8": 0.756372995216946, + "pass@16": 0.794715859617898, + "pass@32": 0.8249475352261741, + "pass@64": 0.8486202280185033, + "pass@128": 0.8674698795180723 + }, + "18": { + "pass@1": 0.48220766129032255, + "pass@2": 0.5944191325882653, + "pass@4": 0.6840774846692548, + "pass@8": 0.7484527607317959, + "pass@16": 0.7932317734924506, + "pass@32": 0.8275928302086873, + "pass@64": 0.8547011083566157, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.325080249244713, + "pass@2": 0.43810883864214883, + "pass@4": 0.5422776692188405, + "pass@8": 0.6265824302323246, + "pass@16": 0.6887125668408031, + "pass@32": 0.7329306621250592, + "pass@64": 0.767905495181644, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.29208896396396394, + "pass@2": 0.401044622378284, + "pass@4": 0.5034210889742385, + "pass@8": 0.5886495943967197, + "pass@16": 0.6554977161404892, + "pass@32": 0.7066888257456404, + "pass@64": 0.746441076359396, + "pass@128": 0.7777777777777778 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..40416c6c1eced2f8c0e5a68130264a040e44c25e --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43e44ac7b210ff5872fa344f195e3a9bc03e0a6919b180b4448547d22b5a7678 +size 2697983399 diff --git a/results/composition/SDAR-100M-block8-epoch4-op11-14/summary.csv b/results/composition/SDAR-100M-block8-epoch4-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..f62b6183b8845b37d35c8982b8db77975e5a5473 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op11-14,0.9930911458333334,0.67023828125,0.8231685855263158,0.0,0.0,0.0,199.22368663194445,370.87193515625,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch4-op11-14/summary.json b/results/composition/SDAR-100M-block8-epoch4-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7589737642c2edd5e189661391ce4d753c86790f --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op11-14", + "id_acc": 0.9930911458333334, + "ood_acc": 0.67023828125, + "total_acc": 0.8231685855263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.22368663194445, + "ood_avg_resp_len": 370.87193515625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 668453, + "answer_accuracy": 0.8255278315681315, + "avg_response_len": 276.6914371245653, + "resp_tokens_sum": 224044804, + "pass_at_k": { + "pass@1": 0.8255278315681315, + "pass@2": 0.861867058770073, + "pass@4": 0.8895697888775553, + "pass@8": 0.9109484207704667, + "pass@16": 0.9271942329187942, + "pass@32": 0.93941210538085, + "pass@64": 0.9488117816403033, + "pass@128": 0.9565286120771419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9550377949852508, + "pass@2": 0.9653236272733609, + "pass@4": 0.9726180897741764, + "pass@8": 0.9783762148389291, + "pass@16": 0.9829116146743994, + "pass@32": 0.9870127565970104, + "pass@64": 0.991180340449138, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997809579439252, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998779296875, + "pass@2": 0.9995659295029528, + "pass@4": 0.9999430012654666, + "pass@8": 0.9999991752021973, + "pass@16": 0.9999999999245107, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982638888888888, + "pass@2": 0.9995289651293588, + "pass@4": 0.9999661601823582, + "pass@8": 0.9999998458721072, + "pass@16": 0.9999999999987073, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989006362275449, + "pass@2": 0.9996018052477722, + "pass@4": 0.9999499412311484, + "pass@8": 0.9999993414788002, + "pass@16": 0.9999999999517831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999075443786982, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9933143028846154, + "pass@2": 0.997447096456693, + "pass@4": 0.999438021268976, + "pass@8": 0.9999627194608325, + "pass@16": 0.9999998418611676, + "pass@32": 0.9999999999993773, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9976620550072086, + "pass@4": 0.9985522935513342, + "pass@8": 0.9993061284706786, + "pass@16": 0.9998463955935609, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9609375, + "pass@2": 0.9711451060649998, + "pass@4": 0.9770741222857058, + "pass@8": 0.9812611025364644, + "pass@16": 0.9842944002686944, + "pass@32": 0.9861828830088263, + "pass@64": 0.9872429473453521, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.9021671963276836, + "pass@2": 0.9215281484941499, + "pass@4": 0.9347171227537232, + "pass@8": 0.9446575312767541, + "pass@16": 0.9525455705033121, + "pass@32": 0.9580749193035334, + "pass@64": 0.9613374661256407, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8692202194357367, + "pass@2": 0.9024653352010464, + "pass@4": 0.9221733035721631, + "pass@8": 0.9359308821175786, + "pass@16": 0.9484843594916188, + "pass@32": 0.960115260632947, + "pass@64": 0.9696726490165624, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8432389018691588, + "pass@2": 0.8816467669929107, + "pass@4": 0.9054283430692655, + "pass@8": 0.9252522133459742, + "pass@16": 0.9410032100908022, + "pass@32": 0.9510890172203004, + "pass@64": 0.9586071337072141, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.8159320809248555, + "pass@2": 0.8758590869782895, + "pass@4": 0.9136971456963835, + "pass@8": 0.9391793646317208, + "pass@16": 0.9568471423736444, + "pass@32": 0.9685935199216499, + "pass@64": 0.9754437045824256, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6567792338709677, + "pass@2": 0.740235664846329, + "pass@4": 0.795893839278154, + "pass@8": 0.8355960749083245, + "pass@16": 0.864558190334898, + "pass@32": 0.8889121743620298, + "pass@64": 0.9101876710408604, + "pass@128": 0.9258064516129032 + }, + "17": { + "pass@1": 0.5896405346820809, + "pass@2": 0.689234175162714, + "pass@4": 0.7562385357538394, + "pass@8": 0.7996128076425166, + "pass@16": 0.8318120005184033, + "pass@32": 0.8596979457171261, + "pass@64": 0.8843409721762664, + "pass@128": 0.9046242774566474 + }, + "18": { + "pass@1": 0.47859375, + "pass@2": 0.582545345894263, + "pass@4": 0.665977828753548, + "pass@8": 0.731456928764178, + "pass@16": 0.780729199135965, + "pass@32": 0.8185404669626503, + "pass@64": 0.848842739464603, + "pass@128": 0.8742857142857143 + }, + "19": { + "pass@1": 0.32244744744744747, + "pass@2": 0.42991527452649503, + "pass@4": 0.5297148988133239, + "pass@8": 0.6158184040059627, + "pass@16": 0.6829310379104327, + "pass@32": 0.7314577265217167, + "pass@64": 0.7668547527202739, + "pass@128": 0.7927927927927928 + }, + "20": { + "pass@1": 0.33054147897897895, + "pass@2": 0.441439224657729, + "pass@4": 0.5459787081794953, + "pass@8": 0.6340915487675354, + "pass@16": 0.7009979125456074, + "pass@32": 0.7479033979165912, + "pass@64": 0.781451169312069, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 678810, + "answer_accuracy": 0.8266879384255651, + "avg_response_len": 285.86639833398283, + "resp_tokens_sum": 234730617, + "pass_at_k": { + "pass@1": 0.8266879384255651, + "pass@2": 0.8628712425049453, + "pass@4": 0.8899870912375201, + "pass@8": 0.9101864400582642, + "pass@16": 0.9256512587008077, + "pass@32": 0.9380929549393652, + "pass@64": 0.9483659509441283, + "pass@128": 0.9568199532346064 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9402319785276073, + "pass@2": 0.952008281604754, + "pass@4": 0.9611208099562711, + "pass@8": 0.967649685144444, + "pass@16": 0.9723239331864337, + "pass@32": 0.976739705975646, + "pass@64": 0.9818104174845861, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999411231884058, + "pass@2": 0.9999315302978432, + "pass@4": 0.9999989373610907, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966869627507163, + "pass@2": 0.9970676905895359, + "pass@4": 0.9971333691633818, + "pass@8": 0.9971346702346441, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9998871026011561, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965889084507042, + "pass@2": 0.9972960796273704, + "pass@4": 0.9975222988851745, + "pass@8": 0.9978309304389954, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932400823353293, + "pass@2": 0.99476379426187, + "pass@4": 0.995734111922387, + "pass@8": 0.9969632799932823, + "pass@16": 0.9984388741597237, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950416, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9623258659638554, + "pass@2": 0.9707566733469312, + "pass@4": 0.9751945817578527, + "pass@8": 0.978528567411368, + "pass@16": 0.9816759959049333, + "pass@32": 0.9840488998900503, + "pass@64": 0.9848917427571217, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9100215517241379, + "pass@2": 0.9266520160195493, + "pass@4": 0.9344253890285655, + "pass@8": 0.9383279633369872, + "pass@16": 0.9410428618385669, + "pass@32": 0.9436759703758117, + "pass@64": 0.9467110995185979, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.8494616596638656, + "pass@2": 0.8807649043869519, + "pass@4": 0.900884443803768, + "pass@8": 0.9155814360333341, + "pass@16": 0.9267145595077136, + "pass@32": 0.9359428582481637, + "pass@64": 0.9445210713800876, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.8588267543859649, + "pass@2": 0.9018694312059676, + "pass@4": 0.9242850279679948, + "pass@8": 0.9383757151048715, + "pass@16": 0.9484554246399113, + "pass@32": 0.9561951272112444, + "pass@64": 0.9627628804004219, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.8121685606060606, + "pass@2": 0.8770117513719882, + "pass@4": 0.915564349342695, + "pass@8": 0.9404922694298705, + "pass@16": 0.9573654432962957, + "pass@32": 0.9699346349116533, + "pass@64": 0.9789174437718314, + "pass@128": 0.9848484848484849 + }, + "16": { + "pass@1": 0.6486823361823362, + "pass@2": 0.7438652500168247, + "pass@4": 0.805919283326763, + "pass@8": 0.8449027197071314, + "pass@16": 0.8727992336539356, + "pass@32": 0.8965191234491018, + "pass@64": 0.9150418318628365, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.5890916149068323, + "pass@2": 0.6926184159045329, + "pass@4": 0.764558463286033, + "pass@8": 0.8095983074474769, + "pass@16": 0.8386906126837259, + "pass@32": 0.8617211263710541, + "pass@64": 0.8851656158646087, + "pass@128": 0.9099378881987578 + }, + "18": { + "pass@1": 0.5270220588235294, + "pass@2": 0.6297294754515979, + "pass@4": 0.7110232550894364, + "pass@8": 0.7729100372599288, + "pass@16": 0.8200711561466233, + "pass@32": 0.8564476725925791, + "pass@64": 0.8842089254988268, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.31570870535714285, + "pass@2": 0.4234084997187852, + "pass@4": 0.5270488630327456, + "pass@8": 0.6161440847378656, + "pass@16": 0.6883030755974877, + "pass@32": 0.7454121901545792, + "pass@64": 0.791591256811504, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.29381081586826346, + "pass@2": 0.39535287201188124, + "pass@4": 0.48994442414383815, + "pass@8": 0.5711483363826257, + "pass@16": 0.6389980310343075, + "pass@32": 0.6950049674288274, + "pass@64": 0.740922892472733, + "pass@128": 0.781437125748503 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 654683, + "answer_accuracy": 0.8171770150982585, + "avg_response_len": 306.3667606147148, + "resp_tokens_sum": 245446343, + "pass_at_k": { + "pass@1": 0.8171770150982585, + "pass@2": 0.8561787954479328, + "pass@4": 0.8856990940619518, + "pass@8": 0.9070705967228174, + "pass@16": 0.9224258546590777, + "pass@32": 0.9339350277080865, + "pass@64": 0.9431577235216402, + "pass@128": 0.9507908611599297 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.962080223880597, + "pass@2": 0.9717545393113174, + "pass@4": 0.9788598931663394, + "pass@8": 0.984093246200315, + "pass@16": 0.9877611821054247, + "pass@32": 0.9904522772205684, + "pass@64": 0.9923508998342762, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999172794117647, + "pass@2": 0.9999478925428441, + "pass@4": 0.9999997033459054, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991867113564669, + "pass@2": 0.9998074964604189, + "pass@4": 0.9999893664711469, + "pass@8": 0.9999999767918476, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9932667042042042, + "pass@2": 0.9964121355607576, + "pass@4": 0.9984610504767983, + "pass@8": 0.999490219467908, + "pass@16": 0.9999277447691266, + "pass@32": 0.999999100781228, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992941117363344, + "pass@2": 0.9948132927817303, + "pass@4": 0.9964856477948294, + "pass@8": 0.9980765642094214, + "pass@16": 0.999336068437543, + "pass@32": 0.9999146790894352, + "pass@64": 0.9999995549101495, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9607886904761904, + "pass@2": 0.9730615704286967, + "pass@4": 0.9794108165050797, + "pass@8": 0.9826995305573731, + "pass@16": 0.9852891085277606, + "pass@32": 0.9874150370684425, + "pass@64": 0.9888440174400547, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9113627675840978, + "pass@2": 0.934330904307833, + "pass@4": 0.950269623407166, + "pass@8": 0.9616482628134414, + "pass@16": 0.9680019758410323, + "pass@32": 0.9717576392340137, + "pass@64": 0.9743729293524362, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8297405478395061, + "pass@2": 0.8684227514095461, + "pass@4": 0.8928393649867837, + "pass@8": 0.9073057009677743, + "pass@16": 0.916060516231829, + "pass@32": 0.9236679890966697, + "pass@64": 0.9314670859752837, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.82870456231454, + "pass@2": 0.87395368466553, + "pass@4": 0.898891906686738, + "pass@8": 0.9156222702891602, + "pass@16": 0.9299421512073494, + "pass@32": 0.9418732831664041, + "pass@64": 0.9518740184036798, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.7886525848765432, + "pass@2": 0.8476004301545641, + "pass@4": 0.8812799615325858, + "pass@8": 0.903221984034905, + "pass@16": 0.9189262347802786, + "pass@32": 0.9304864083273199, + "pass@64": 0.9390019202952004, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.634010877581121, + "pass@2": 0.740449271827747, + "pass@4": 0.8125692819924054, + "pass@8": 0.8575554027858113, + "pass@16": 0.8848759001996248, + "pass@32": 0.9038438545841817, + "pass@64": 0.9216124078198928, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.517554593373494, + "pass@2": 0.6258230510625178, + "pass@4": 0.7037033872271984, + "pass@8": 0.756372995216946, + "pass@16": 0.794715859617898, + "pass@32": 0.8249475352261741, + "pass@64": 0.8486202280185033, + "pass@128": 0.8674698795180723 + }, + "18": { + "pass@1": 0.48220766129032255, + "pass@2": 0.5944191325882653, + "pass@4": 0.6840774846692548, + "pass@8": 0.7484527607317959, + "pass@16": 0.7932317734924506, + "pass@32": 0.8275928302086873, + "pass@64": 0.8547011083566157, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.325080249244713, + "pass@2": 0.43810883864214883, + "pass@4": 0.5422776692188405, + "pass@8": 0.6265824302323246, + "pass@16": 0.6887125668408031, + "pass@32": 0.7329306621250592, + "pass@64": 0.767905495181644, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.29208896396396394, + "pass@2": 0.401044622378284, + "pass@4": 0.5034210889742385, + "pass@8": 0.5886495943967197, + "pass@16": 0.6554977161404892, + "pass@32": 0.7066888257456404, + "pass@64": 0.746441076359396, + "pass@128": 0.7777777777777778 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7034995cbfc8647263590456dada2fac0ecfcfab --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e56ada2819d1c755479e924cfbbe397f48aeb1186995572a1cf6e5407a55c84 +size 1789719521 diff --git a/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_metrics.json b/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a20cd7c7f788ecd06b25a4ec8dadba72e8522c0b --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op17-20", + "id": { + "answer_accuracy": 0.9918411458333334, + "avg_response_len": 199.07718576388888, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9918411458333334, + "pass@2": 0.9942585848643914, + "pass@4": 0.9959380943319587, + "pass@8": 0.9971967379646093, + "pass@16": 0.9981141062657112, + "pass@32": 0.9987434866069741, + "pass@64": 0.9990930305192568, + "pass@128": 0.9992222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465625, + "pass@2": 0.9594614911417327, + "pass@4": 0.9697060062804653, + "pass@8": 0.9780383427703782, + "pass@16": 0.9844741779786856, + "pass@32": 0.9891185169487656, + "pass@64": 0.9918733630572701, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999921875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999609375, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999375, + "pass@2": 0.9999986466535433, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9977421875, + "pass@2": 0.9991828248031493, + "pass@4": 0.9998286774465691, + "pass@8": 0.9999892946880925, + "pass@16": 0.9999999537118734, + "pass@32": 0.9999999999998068, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.996171875, + "pass@2": 0.9974132627952755, + "pass@4": 0.998452184008249, + "pass@8": 0.9991251858828614, + "pass@16": 0.9994832503889618, + "pass@32": 0.9997689348631511, + "pass@64": 0.9999711811023585, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9988125, + "pass@2": 0.9995367864173228, + "pass@4": 0.9998419919385075, + "pass@8": 0.9999775006585908, + "pass@16": 0.999999640100668, + "pass@32": 0.9999999999754914, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993953125, + "pass@2": 0.9971211860236221, + "pass@4": 0.9987266791338584, + "pass@8": 0.9995431057153819, + "pass@16": 0.9999044500192191, + "pass@32": 0.9999949205487579, + "pass@64": 0.9999999977569968, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935078125, + "pass@2": 0.9956178641732286, + "pass@4": 0.9968873224596928, + "pass@8": 0.9980972119661831, + "pass@16": 0.999165484191994, + "pass@32": 0.9998090071267925, + "pass@64": 0.9999927327566851, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9465625, + "2": 0.999921875, + "3": 0.9999609375, + "4": 0.9999375, + "5": 0.9977421875, + "6": 0.996171875, + "7": 0.9988125, + "8": 0.993953125, + "9": 0.9935078125 + }, + "per_op_avg_response_len": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375880, + "answer_accuracy": 0.9917468760553867, + "avg_response_len": 189.6703235815603, + "resp_tokens_sum": 71886570, + "pass_at_k": { + "pass@1": 0.9917468760553867, + "pass@2": 0.9946696739370348, + "pass@4": 0.996500682158904, + "pass@8": 0.9976780569502949, + "pass@16": 0.9983965809007429, + "pass@32": 0.9987782425175514, + "pass@64": 0.9989567445453673, + "pass@128": 0.9989868287740629 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.947686209439528, + "pass@2": 0.9617538992636985, + "pass@4": 0.9725950904367042, + "pass@8": 0.9805598544320808, + "pass@16": 0.9860735851683115, + "pass@32": 0.9893290501813384, + "pass@64": 0.9908876713836634, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9997566199376947, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657481, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978422619047619, + "pass@2": 0.9992364235720536, + "pass@4": 0.9998443864159838, + "pass@8": 0.9999934611094885, + "pass@16": 0.999999993102673, + "pass@32": 0.9999999999999994, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9969385785515584, + "pass@4": 0.9984315971543977, + "pass@8": 0.9995862366315358, + "pass@16": 0.9999714614533096, + "pass@32": 0.999999909391957, + "pass@64": 0.9999999999999891, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987056213017751, + "pass@2": 0.9998150887573964, + "pass@4": 0.9999939333581823, + "pass@8": 0.9999999935548729, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927634214743589, + "pass@2": 0.997842611169998, + "pass@4": 0.9996803302231451, + "pass@8": 0.9999892239856973, + "pass@16": 0.9999999910956074, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951144366197183, + "pass@2": 0.9977098813352555, + "pass@4": 0.9988829544194299, + "pass@8": 0.999601568979854, + "pass@16": 0.999951739437933, + "pass@32": 0.9999996009748804, + "pass@64": 0.9999999999993653, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387187, + "answer_accuracy": 0.9911200647116645, + "avg_response_len": 197.10492351327, + "resp_tokens_sum": 77000221, + "pass_at_k": { + "pass@1": 0.9911200647116645, + "pass@2": 0.9931661889454186, + "pass@4": 0.9947510570742057, + "pass@8": 0.9961194415684201, + "pass@16": 0.9973336875489526, + "pass@32": 0.9984023042996628, + "pass@64": 0.9990767801528179, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9353671395705522, + "pass@2": 0.9489883489928022, + "pass@4": 0.9602921086167909, + "pass@8": 0.970010030193041, + "pass@16": 0.9784615849176065, + "pass@32": 0.9861956753511112, + "pass@64": 0.9914655018647149, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968297101449275, + "pass@2": 0.9986281096656396, + "pass@4": 0.9996703835933552, + "pass@8": 0.9999750963168695, + "pass@16": 0.9999998721310288, + "pass@32": 0.9999999999994407, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955229226361032, + "pass@2": 0.9963235086749543, + "pass@4": 0.9971980834952937, + "pass@8": 0.9978938143484848, + "pass@16": 0.9985466574181527, + "pass@32": 0.9993380089577011, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998193641618497, + "pass@2": 0.9999946662646215, + "pass@4": 0.9999999959362016, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9978419235887768, + "pass@4": 0.9984183204212148, + "pass@8": 0.9991300714136, + "pass@16": 0.9997538434444305, + "pass@32": 0.9999857247050008, + "pass@64": 0.9999999936816814, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9949026651421564, + "pass@4": 0.9957427633484436, + "pass@8": 0.9969632965123278, + "pass@16": 0.9984388741597334, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950417, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379534, + "answer_accuracy": 0.9926713675929025, + "avg_response_len": 210.41734756863073, + "resp_tokens_sum": 80450127, + "pass_at_k": { + "pass@1": 0.9926713675929025, + "pass@2": 0.9949672415770174, + "pass@4": 0.9965932718194292, + "pass@8": 0.9978203479694808, + "pass@16": 0.9986314917793431, + "pass@32": 0.9990576399885253, + "pass@64": 0.9992447341975484, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9673334851333878, + "pass@4": 0.975943412577159, + "pass@8": 0.9832993500745408, + "pass@16": 0.9887067340999656, + "pass@32": 0.9917497873815814, + "pass@64": 0.9932677279113761, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985753676470588, + "pass@2": 0.9996960398332562, + "pass@4": 0.9999747452524317, + "pass@8": 0.9999998417920103, + "pass@16": 0.9999999999980195, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974122634069401, + "pass@2": 0.9991131641372115, + "pass@4": 0.999854572299125, + "pass@8": 0.9999950840388245, + "pass@16": 0.9999999962921804, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978243670886076, + "pass@2": 0.9987377591448221, + "pass@4": 0.9995064679415073, + "pass@8": 0.999928806446341, + "pass@16": 0.9999988610780643, + "pass@32": 0.9999999999224412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920232732732732, + "pass@2": 0.9956769022959969, + "pass@4": 0.9981618989630802, + "pass@8": 0.9995654428829316, + "pass@16": 0.9999754906144646, + "pass@32": 0.9999999648002483, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916097266881029, + "pass@2": 0.9939979618705219, + "pass@4": 0.9958385552489217, + "pass@8": 0.9975977940296384, + "pass@16": 0.9990483367272565, + "pass@32": 0.999836024498138, + "pass@64": 0.9999978527259372, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6973953125, + "avg_response_len": 374.408946875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6973953125, + "pass@2": 0.7655862696850245, + "pass@4": 0.814535287570296, + "pass@8": 0.8507059550506876, + "pass@16": 0.8780092767935522, + "pass@32": 0.8989192866811604, + "pass@64": 0.9154332057276459, + "pass@128": 0.9292 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.949765625, + "pass@2": 0.9636034694881894, + "pass@4": 0.9714430826771656, + "pass@8": 0.976761624983819, + "pass@16": 0.9811986254788247, + "pass@32": 0.9850637024974109, + "pass@64": 0.9880005694807404, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8930546875, + "pass@2": 0.9174040354330714, + "pass@4": 0.9324833617360326, + "pass@8": 0.9431630390274265, + "pass@16": 0.9506346812330824, + "pass@32": 0.9557801855766974, + "pass@64": 0.9596686566076377, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.816203125, + "pass@2": 0.8637347440944891, + "pass@4": 0.8925301711661043, + "pass@8": 0.9116243283252289, + "pass@16": 0.926667277462485, + "pass@32": 0.9394301326422876, + "pass@64": 0.9501194331153558, + "pass@128": 0.959 + }, + "14": { + "pass@1": 0.832890625, + "pass@2": 0.8779275344488201, + "pass@4": 0.9027908765466814, + "pass@8": 0.9202558362172919, + "pass@16": 0.9348536657941723, + "pass@32": 0.9468987710530873, + "pass@64": 0.9561237781715934, + "pass@128": 0.963 + }, + "15": { + "pass@1": 0.8093046875, + "pass@2": 0.8665110728346466, + "pass@4": 0.9013680777090353, + "pass@8": 0.9241231624779181, + "pass@16": 0.9395981544845722, + "pass@32": 0.9511624667281456, + "pass@64": 0.9609580955836557, + "pass@128": 0.97 + }, + "16": { + "pass@1": 0.6662734375, + "pass@2": 0.7528844734251962, + "pass@4": 0.8093653296775412, + "pass@8": 0.8483589019735052, + "pass@16": 0.8776700227907909, + "pass@32": 0.9009114846997556, + "pass@64": 0.9198523641648548, + "pass@128": 0.937 + }, + "17": { + "pass@1": 0.61553125, + "pass@2": 0.7092402805118122, + "pass@4": 0.7726679700037498, + "pass@8": 0.8156043454313336, + "pass@16": 0.8469887135917317, + "pass@32": 0.872209664689536, + "pass@64": 0.8923636305747241, + "pass@128": 0.909 + }, + "18": { + "pass@1": 0.5731796875, + "pass@2": 0.6679947096456698, + "pass@4": 0.7395980508998866, + "pass@8": 0.7930719507635707, + "pass@16": 0.8322808994809556, + "pass@32": 0.8617942420278873, + "pass@64": 0.8851850179047893, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.397671875, + "pass@2": 0.5062367125984255, + "pass@4": 0.6007368568616425, + "pass@8": 0.6788454836198332, + "pass@16": 0.7393707639669304, + "pass@32": 0.7840548102990779, + "pass@64": 0.8199610949104585, + "pass@128": 0.853 + }, + "20": { + "pass@1": 0.420078125, + "pass@2": 0.5303256643700788, + "pass@4": 0.6223690984251962, + "pass@8": 0.6952508776870037, + "pass@16": 0.7508299636518877, + "pass@32": 0.7918874065976513, + "pass@64": 0.8220994167625919, + "pass@128": 0.844 + } + }, + "per_op_accuracy": { + "11": 0.949765625, + "12": 0.8930546875, + "13": 0.816203125, + "14": 0.832890625, + "15": 0.8093046875, + "16": 0.6662734375, + "17": 0.61553125, + "18": 0.5731796875, + "19": 0.397671875, + "20": 0.420078125 + }, + "per_op_avg_response_len": { + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 302613, + "answer_accuracy": 0.7025747585438336, + "avg_response_len": 356.7563614413076, + "resp_tokens_sum": 153662100, + "pass_at_k": { + "pass@1": 0.7025747585438336, + "pass@2": 0.7702944565993196, + "pass@4": 0.8190666393112498, + "pass@8": 0.8552391822468836, + "pass@16": 0.8823689747494595, + "pass@32": 0.9028818370439949, + "pass@64": 0.9189488249813886, + "pass@128": 0.9325408618127786 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9421033994334278, + "pass@2": 0.9585153827708507, + "pass@4": 0.9683495726277837, + "pass@8": 0.9750743355007809, + "pass@16": 0.9806874497365086, + "pass@32": 0.9855674217757143, + "pass@64": 0.9892681802260217, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8856594279661016, + "pass@2": 0.910248176075448, + "pass@4": 0.9253412335640251, + "pass@8": 0.9357745885481913, + "pass@16": 0.9435988842897304, + "pass@32": 0.9494690435475064, + "pass@64": 0.9538236623937969, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8312842868338558, + "pass@2": 0.8800130513662283, + "pass@4": 0.9112304953847852, + "pass@8": 0.9301201681225754, + "pass@16": 0.9426301334874778, + "pass@32": 0.9531258102406001, + "pass@64": 0.962334874645846, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.8299260124610592, + "pass@2": 0.8766005592758852, + "pass@4": 0.9035752581161002, + "pass@8": 0.9220158894644768, + "pass@16": 0.9356224843327265, + "pass@32": 0.945334584744341, + "pass@64": 0.953042570524806, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.8186867774566474, + "pass@2": 0.877645532747713, + "pass@4": 0.9140065072248914, + "pass@8": 0.937924646916376, + "pass@16": 0.9541987911233919, + "pass@32": 0.965099381921458, + "pass@64": 0.9726851787993989, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6638860887096775, + "pass@2": 0.7459883794767583, + "pass@4": 0.8018100526748667, + "pass@8": 0.8433015429764784, + "pass@16": 0.8753356035074012, + "pass@32": 0.9009404640966499, + "pass@64": 0.9223832471129497, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6308932442196532, + "pass@2": 0.7216910216876794, + "pass@4": 0.7815059672815464, + "pass@8": 0.8225626990224884, + "pass@16": 0.8534324527753754, + "pass@32": 0.8779455950894322, + "pass@64": 0.8973389717015748, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.551875, + "pass@2": 0.6496741422947138, + "pass@4": 0.7255719232417374, + "pass@8": 0.7824379510059899, + "pass@16": 0.823303427941537, + "pass@32": 0.8522402739187724, + "pass@64": 0.8735607870746009, + "pass@128": 0.8914285714285715 + }, + "19": { + "pass@1": 0.4078688063063063, + "pass@2": 0.5134215169894304, + "pass@4": 0.6041881234440287, + "pass@8": 0.6818408987273152, + "pass@16": 0.7434984007892304, + "pass@32": 0.7890733463330541, + "pass@64": 0.8258918288508185, + "pass@128": 0.8558558558558559 + }, + "20": { + "pass@1": 0.4509196696696697, + "pass@2": 0.5596455215057575, + "pass@4": 0.6476805069974355, + "pass@8": 0.7155572588224337, + "pass@16": 0.7669233647580539, + "pass@32": 0.8067563935871334, + "pass@64": 0.837089022951731, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 302599, + "answer_accuracy": 0.7029600617008623, + "avg_response_len": 369.36367965730005, + "resp_tokens_sum": 158997767, + "pass_at_k": { + "pass@1": 0.7029600617008623, + "pass@2": 0.7702698688366508, + "pass@4": 0.8183143234994847, + "pass@8": 0.8539573341142721, + "pass@16": 0.8813696069826088, + "pass@32": 0.9028034603918524, + "pass@64": 0.9200449007829323, + "pass@128": 0.9345822182575082 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9559723268072289, + "pass@2": 0.966372008704108, + "pass@4": 0.9713946641458974, + "pass@8": 0.9749261474893676, + "pass@16": 0.9779525234791777, + "pass@32": 0.9806901170607644, + "pass@64": 0.9830740245540501, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.895694553291536, + "pass@2": 0.9185176671685638, + "pass@4": 0.9316084925121034, + "pass@8": 0.9394861999416354, + "pass@16": 0.9443789712253488, + "pass@32": 0.9476920343833845, + "pass@64": 0.9504452941632805, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8206188725490197, + "pass@2": 0.8654555818390349, + "pass@4": 0.8914457194951465, + "pass@8": 0.91024432511722, + "pass@16": 0.9260177446913851, + "pass@32": 0.9387626126789113, + "pass@64": 0.9509378123272939, + "pass@128": 0.9635854341736695 + }, + "14": { + "pass@1": 0.8450520833333334, + "pass@2": 0.892094902610858, + "pass@4": 0.9161785703322166, + "pass@8": 0.9328964288192578, + "pass@16": 0.9477363031164306, + "pass@32": 0.960031109634046, + "pass@64": 0.9693199907235079, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8125236742424242, + "pass@2": 0.8730982313290383, + "pass@4": 0.9092206175932549, + "pass@8": 0.9321373846919218, + "pass@16": 0.947976048382885, + "pass@32": 0.9615416739411536, + "pass@64": 0.9736111224580455, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6658653846153846, + "pass@2": 0.75107889158086, + "pass@4": 0.807817280051532, + "pass@8": 0.8455472972839031, + "pass@16": 0.8726949869908102, + "pass@32": 0.8942385463597932, + "pass@64": 0.9125603746766824, + "pass@128": 0.9287749287749287 + }, + "17": { + "pass@1": 0.6382472826086957, + "pass@2": 0.7308938199980439, + "pass@4": 0.7938274872085086, + "pass@8": 0.8379544601446831, + "pass@16": 0.8705400266432815, + "pass@32": 0.8951335125041588, + "pass@64": 0.9145615683410627, + "pass@128": 0.9347826086956522 + }, + "18": { + "pass@1": 0.597219669117647, + "pass@2": 0.6891023043075495, + "pass@4": 0.7591783864333136, + "pass@8": 0.8121559273453292, + "pass@16": 0.8524030793941533, + "pass@32": 0.8851876403918829, + "pass@64": 0.9114327068575712, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.39669363839285715, + "pass@2": 0.5050988499484439, + "pass@4": 0.5999355119672539, + "pass@8": 0.6770325751872642, + "pass@16": 0.7367422561935473, + "pass@32": 0.7810862819828637, + "pass@64": 0.8162268287403831, + "pass@128": 0.8511904761904762 + }, + "20": { + "pass@1": 0.4049869011976048, + "pass@2": 0.5131853664717809, + "pass@4": 0.6042743593240965, + "pass@8": 0.6785910123757385, + "pass@16": 0.738234545184363, + "pass@32": 0.7842155855252223, + "pass@64": 0.8183162239457017, + "pass@128": 0.8413173652694611 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 287454, + "answer_accuracy": 0.6863491366136919, + "avg_response_len": 397.7488562996638, + "resp_tokens_sum": 166583585, + "pass_at_k": { + "pass@1": 0.6863491366136919, + "pass@2": 0.7559304038802191, + "pass@4": 0.8059910038178307, + "pass@8": 0.8427020744559618, + "pass@16": 0.8700718764122981, + "pass@32": 0.890851909492874, + "pass@64": 0.9070777077875871, + "pass@128": 0.9202322738386308 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9518105158730159, + "pass@2": 0.9663874046994128, + "pass@4": 0.974960809660697, + "pass@8": 0.9805869955097556, + "pass@16": 0.9851927553230163, + "pass@32": 0.98910883729019, + "pass@64": 0.9917724625682862, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8984852828746177, + "pass@2": 0.9240643586409498, + "pass@4": 0.941068672608584, + "pass@8": 0.9547484308868067, + "pass@16": 0.964354080653309, + "pass@32": 0.9705027070109506, + "pass@64": 0.9749939794561082, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7964891975308642, + "pass@2": 0.8458115370127349, + "pass@4": 0.8753133372795989, + "pass@8": 0.8949344772446929, + "pass@16": 0.9116664506949259, + "pass@32": 0.9266813162011269, + "pass@64": 0.9371907688348377, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.8233725890207715, + "pass@2": 0.864813941330405, + "pass@4": 0.8884574113881165, + "pass@8": 0.9057512078427568, + "pass@16": 0.9210475746514785, + "pass@32": 0.9350615129237696, + "pass@64": 0.9456666949723773, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7960069444444444, + "pass@2": 0.8479114264362789, + "pass@4": 0.8798735259365724, + "pass@8": 0.9012219064707329, + "pass@16": 0.9154730765108868, + "pass@32": 0.9257078030331487, + "pass@64": 0.9355473867528037, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6688790560471977, + "pass@2": 0.761060132278819, + "pass@4": 0.8178771328915742, + "pass@8": 0.8558947560594331, + "pass@16": 0.8849558857513954, + "pass@32": 0.9077941329129988, + "pass@64": 0.9250881004375349, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.5774896460843374, + "pass@2": 0.6752631836400717, + "pass@4": 0.7429351037445621, + "pass@8": 0.7866756488041113, + "pass@16": 0.8174313143142055, + "pass@32": 0.8439984872357021, + "pass@64": 0.8656491607233635, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.570866935483871, + "pass@2": 0.6655289560579124, + "pass@4": 0.7339587947675894, + "pass@8": 0.7841472664969731, + "pass@16": 0.8203472667980849, + "pass@32": 0.8469237045905657, + "pass@64": 0.8695213616035634, + "pass@128": 0.8903225806451613 + }, + "19": { + "pass@1": 0.3884063444108761, + "pass@2": 0.5001635463996004, + "pass@4": 0.5980781865068977, + "pass@8": 0.6776722630837357, + "pass@16": 0.7378864000697488, + "pass@32": 0.782019320918216, + "pass@64": 0.81778520074431, + "pass@128": 0.851963746223565 + }, + "20": { + "pass@1": 0.4043731231231231, + "pass@2": 0.5181975774987587, + "pass@4": 0.6152067675099175, + "pass@8": 0.6916543914883966, + "pass@16": 0.7473698050927324, + "pass@32": 0.7847132791823201, + "pass@64": 0.8109043643297321, + "pass@128": 0.8288288288288288 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8368696546052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8368696546052632, + "pass@2": 0.8739047347699785, + "pass@4": 0.9004629328784418, + "pass@8": 0.920096325904657, + "pass@16": 0.9349010381224566, + "pass@32": 0.9462044340144459, + "pass@64": 0.9550615437868266, + "pass@128": 0.9623684210526315 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465625, + "pass@2": 0.9594614911417327, + "pass@4": 0.9697060062804653, + "pass@8": 0.9780383427703782, + "pass@16": 0.9844741779786856, + "pass@32": 0.9891185169487656, + "pass@64": 0.9918733630572701, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999921875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999609375, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999375, + "pass@2": 0.9999986466535433, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9977421875, + "pass@2": 0.9991828248031493, + "pass@4": 0.9998286774465691, + "pass@8": 0.9999892946880925, + "pass@16": 0.9999999537118734, + "pass@32": 0.9999999999998068, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.996171875, + "pass@2": 0.9974132627952755, + "pass@4": 0.998452184008249, + "pass@8": 0.9991251858828614, + "pass@16": 0.9994832503889618, + "pass@32": 0.9997689348631511, + "pass@64": 0.9999711811023585, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9988125, + "pass@2": 0.9995367864173228, + "pass@4": 0.9998419919385075, + "pass@8": 0.9999775006585908, + "pass@16": 0.999999640100668, + "pass@32": 0.9999999999754914, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993953125, + "pass@2": 0.9971211860236221, + "pass@4": 0.9987266791338584, + "pass@8": 0.9995431057153819, + "pass@16": 0.9999044500192191, + "pass@32": 0.9999949205487579, + "pass@64": 0.9999999977569968, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935078125, + "pass@2": 0.9956178641732286, + "pass@4": 0.9968873224596928, + "pass@8": 0.9980972119661831, + "pass@16": 0.999165484191994, + "pass@32": 0.9998090071267925, + "pass@64": 0.9999927327566851, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.949765625, + "pass@2": 0.9636034694881894, + "pass@4": 0.9714430826771656, + "pass@8": 0.976761624983819, + "pass@16": 0.9811986254788247, + "pass@32": 0.9850637024974109, + "pass@64": 0.9880005694807404, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8930546875, + "pass@2": 0.9174040354330714, + "pass@4": 0.9324833617360326, + "pass@8": 0.9431630390274265, + "pass@16": 0.9506346812330824, + "pass@32": 0.9557801855766974, + "pass@64": 0.9596686566076377, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.816203125, + "pass@2": 0.8637347440944891, + "pass@4": 0.8925301711661043, + "pass@8": 0.9116243283252289, + "pass@16": 0.926667277462485, + "pass@32": 0.9394301326422876, + "pass@64": 0.9501194331153558, + "pass@128": 0.959 + }, + "14": { + "pass@1": 0.832890625, + "pass@2": 0.8779275344488201, + "pass@4": 0.9027908765466814, + "pass@8": 0.9202558362172919, + "pass@16": 0.9348536657941723, + "pass@32": 0.9468987710530873, + "pass@64": 0.9561237781715934, + "pass@128": 0.963 + }, + "15": { + "pass@1": 0.8093046875, + "pass@2": 0.8665110728346466, + "pass@4": 0.9013680777090353, + "pass@8": 0.9241231624779181, + "pass@16": 0.9395981544845722, + "pass@32": 0.9511624667281456, + "pass@64": 0.9609580955836557, + "pass@128": 0.97 + }, + "16": { + "pass@1": 0.6662734375, + "pass@2": 0.7528844734251962, + "pass@4": 0.8093653296775412, + "pass@8": 0.8483589019735052, + "pass@16": 0.8776700227907909, + "pass@32": 0.9009114846997556, + "pass@64": 0.9198523641648548, + "pass@128": 0.937 + }, + "17": { + "pass@1": 0.61553125, + "pass@2": 0.7092402805118122, + "pass@4": 0.7726679700037498, + "pass@8": 0.8156043454313336, + "pass@16": 0.8469887135917317, + "pass@32": 0.872209664689536, + "pass@64": 0.8923636305747241, + "pass@128": 0.909 + }, + "18": { + "pass@1": 0.5731796875, + "pass@2": 0.6679947096456698, + "pass@4": 0.7395980508998866, + "pass@8": 0.7930719507635707, + "pass@16": 0.8322808994809556, + "pass@32": 0.8617942420278873, + "pass@64": 0.8851850179047893, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.397671875, + "pass@2": 0.5062367125984255, + "pass@4": 0.6007368568616425, + "pass@8": 0.6788454836198332, + "pass@16": 0.7393707639669304, + "pass@32": 0.7840548102990779, + "pass@64": 0.8199610949104585, + "pass@128": 0.853 + }, + "20": { + "pass@1": 0.420078125, + "pass@2": 0.5303256643700788, + "pass@4": 0.6223690984251962, + "pass@8": 0.6952508776870037, + "pass@16": 0.7508299636518877, + "pass@32": 0.7918874065976513, + "pass@64": 0.8220994167625919, + "pass@128": 0.844 + } + }, + "per_op_accuracy": { + "10": 0.9465625, + "2": 0.999921875, + "3": 0.9999609375, + "4": 0.9999375, + "5": 0.9977421875, + "6": 0.996171875, + "7": 0.9988125, + "8": 0.993953125, + "9": 0.9935078125, + "11": 0.949765625, + "12": 0.8930546875, + "13": 0.816203125, + "14": 0.832890625, + "15": 0.8093046875, + "16": 0.6662734375, + "17": 0.61553125, + "18": 0.5731796875, + "19": 0.397671875, + "20": 0.420078125 + }, + "per_op_avg_response_len": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625, + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625, + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 678493, + "answer_accuracy": 0.8379270569870376, + "avg_response_len": 278.5486854845084, + "resp_tokens_sum": 225548670, + "pass_at_k": { + "pass@1": 0.8379270569870376, + "pass@2": 0.8753173808068597, + "pass@4": 0.9021178882634914, + "pass@8": 0.9219103027016352, + "pass@16": 0.936677817906899, + "pass@32": 0.9477679035326472, + "pass@64": 0.9563979950460396, + "pass@128": 0.9636421119190641 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.947686209439528, + "pass@2": 0.9617538992636985, + "pass@4": 0.9725950904367042, + "pass@8": 0.9805598544320808, + "pass@16": 0.9860735851683115, + "pass@32": 0.9893290501813384, + "pass@64": 0.9908876713836634, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9997566199376947, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657481, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978422619047619, + "pass@2": 0.9992364235720536, + "pass@4": 0.9998443864159838, + "pass@8": 0.9999934611094885, + "pass@16": 0.999999993102673, + "pass@32": 0.9999999999999994, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9969385785515584, + "pass@4": 0.9984315971543977, + "pass@8": 0.9995862366315358, + "pass@16": 0.9999714614533096, + "pass@32": 0.999999909391957, + "pass@64": 0.9999999999999891, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987056213017751, + "pass@2": 0.9998150887573964, + "pass@4": 0.9999939333581823, + "pass@8": 0.9999999935548729, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927634214743589, + "pass@2": 0.997842611169998, + "pass@4": 0.9996803302231451, + "pass@8": 0.9999892239856973, + "pass@16": 0.9999999910956074, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951144366197183, + "pass@2": 0.9977098813352555, + "pass@4": 0.9988829544194299, + "pass@8": 0.999601568979854, + "pass@16": 0.999951739437933, + "pass@32": 0.9999996009748804, + "pass@64": 0.9999999999993653, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9421033994334278, + "pass@2": 0.9585153827708507, + "pass@4": 0.9683495726277837, + "pass@8": 0.9750743355007809, + "pass@16": 0.9806874497365086, + "pass@32": 0.9855674217757143, + "pass@64": 0.9892681802260217, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8856594279661016, + "pass@2": 0.910248176075448, + "pass@4": 0.9253412335640251, + "pass@8": 0.9357745885481913, + "pass@16": 0.9435988842897304, + "pass@32": 0.9494690435475064, + "pass@64": 0.9538236623937969, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8312842868338558, + "pass@2": 0.8800130513662283, + "pass@4": 0.9112304953847852, + "pass@8": 0.9301201681225754, + "pass@16": 0.9426301334874778, + "pass@32": 0.9531258102406001, + "pass@64": 0.962334874645846, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.8299260124610592, + "pass@2": 0.8766005592758852, + "pass@4": 0.9035752581161002, + "pass@8": 0.9220158894644768, + "pass@16": 0.9356224843327265, + "pass@32": 0.945334584744341, + "pass@64": 0.953042570524806, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.8186867774566474, + "pass@2": 0.877645532747713, + "pass@4": 0.9140065072248914, + "pass@8": 0.937924646916376, + "pass@16": 0.9541987911233919, + "pass@32": 0.965099381921458, + "pass@64": 0.9726851787993989, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6638860887096775, + "pass@2": 0.7459883794767583, + "pass@4": 0.8018100526748667, + "pass@8": 0.8433015429764784, + "pass@16": 0.8753356035074012, + "pass@32": 0.9009404640966499, + "pass@64": 0.9223832471129497, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6308932442196532, + "pass@2": 0.7216910216876794, + "pass@4": 0.7815059672815464, + "pass@8": 0.8225626990224884, + "pass@16": 0.8534324527753754, + "pass@32": 0.8779455950894322, + "pass@64": 0.8973389717015748, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.551875, + "pass@2": 0.6496741422947138, + "pass@4": 0.7255719232417374, + "pass@8": 0.7824379510059899, + "pass@16": 0.823303427941537, + "pass@32": 0.8522402739187724, + "pass@64": 0.8735607870746009, + "pass@128": 0.8914285714285715 + }, + "19": { + "pass@1": 0.4078688063063063, + "pass@2": 0.5134215169894304, + "pass@4": 0.6041881234440287, + "pass@8": 0.6818408987273152, + "pass@16": 0.7434984007892304, + "pass@32": 0.7890733463330541, + "pass@64": 0.8258918288508185, + "pass@128": 0.8558558558558559 + }, + "20": { + "pass@1": 0.4509196696696697, + "pass@2": 0.5596455215057575, + "pass@4": 0.6476805069974355, + "pass@8": 0.7155572588224337, + "pass@16": 0.7669233647580539, + "pass@32": 0.8067563935871334, + "pass@64": 0.837089022951731, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689786, + "answer_accuracy": 0.8400550467653936, + "avg_response_len": 287.4098645752143, + "resp_tokens_sum": 235997988, + "pass_at_k": { + "pass@1": 0.8400550467653936, + "pass@2": 0.8763150081931377, + "pass@4": 0.9022558528634762, + "pass@8": 0.9215923694922954, + "pass@16": 0.9365406707220455, + "pass@32": 0.9482855604084754, + "pass@64": 0.9576451027840119, + "pass@128": 0.9653936087295402 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9353671395705522, + "pass@2": 0.9489883489928022, + "pass@4": 0.9602921086167909, + "pass@8": 0.970010030193041, + "pass@16": 0.9784615849176065, + "pass@32": 0.9861956753511112, + "pass@64": 0.9914655018647149, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968297101449275, + "pass@2": 0.9986281096656396, + "pass@4": 0.9996703835933552, + "pass@8": 0.9999750963168695, + "pass@16": 0.9999998721310288, + "pass@32": 0.9999999999994407, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955229226361032, + "pass@2": 0.9963235086749543, + "pass@4": 0.9971980834952937, + "pass@8": 0.9978938143484848, + "pass@16": 0.9985466574181527, + "pass@32": 0.9993380089577011, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998193641618497, + "pass@2": 0.9999946662646215, + "pass@4": 0.9999999959362016, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9978419235887768, + "pass@4": 0.9984183204212148, + "pass@8": 0.9991300714136, + "pass@16": 0.9997538434444305, + "pass@32": 0.9999857247050008, + "pass@64": 0.9999999936816814, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9949026651421564, + "pass@4": 0.9957427633484436, + "pass@8": 0.9969632965123278, + "pass@16": 0.9984388741597334, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950417, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9559723268072289, + "pass@2": 0.966372008704108, + "pass@4": 0.9713946641458974, + "pass@8": 0.9749261474893676, + "pass@16": 0.9779525234791777, + "pass@32": 0.9806901170607644, + "pass@64": 0.9830740245540501, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.895694553291536, + "pass@2": 0.9185176671685638, + "pass@4": 0.9316084925121034, + "pass@8": 0.9394861999416354, + "pass@16": 0.9443789712253488, + "pass@32": 0.9476920343833845, + "pass@64": 0.9504452941632805, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8206188725490197, + "pass@2": 0.8654555818390349, + "pass@4": 0.8914457194951465, + "pass@8": 0.91024432511722, + "pass@16": 0.9260177446913851, + "pass@32": 0.9387626126789113, + "pass@64": 0.9509378123272939, + "pass@128": 0.9635854341736695 + }, + "14": { + "pass@1": 0.8450520833333334, + "pass@2": 0.892094902610858, + "pass@4": 0.9161785703322166, + "pass@8": 0.9328964288192578, + "pass@16": 0.9477363031164306, + "pass@32": 0.960031109634046, + "pass@64": 0.9693199907235079, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8125236742424242, + "pass@2": 0.8730982313290383, + "pass@4": 0.9092206175932549, + "pass@8": 0.9321373846919218, + "pass@16": 0.947976048382885, + "pass@32": 0.9615416739411536, + "pass@64": 0.9736111224580455, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6658653846153846, + "pass@2": 0.75107889158086, + "pass@4": 0.807817280051532, + "pass@8": 0.8455472972839031, + "pass@16": 0.8726949869908102, + "pass@32": 0.8942385463597932, + "pass@64": 0.9125603746766824, + "pass@128": 0.9287749287749287 + }, + "17": { + "pass@1": 0.6382472826086957, + "pass@2": 0.7308938199980439, + "pass@4": 0.7938274872085086, + "pass@8": 0.8379544601446831, + "pass@16": 0.8705400266432815, + "pass@32": 0.8951335125041588, + "pass@64": 0.9145615683410627, + "pass@128": 0.9347826086956522 + }, + "18": { + "pass@1": 0.597219669117647, + "pass@2": 0.6891023043075495, + "pass@4": 0.7591783864333136, + "pass@8": 0.8121559273453292, + "pass@16": 0.8524030793941533, + "pass@32": 0.8851876403918829, + "pass@64": 0.9114327068575712, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.39669363839285715, + "pass@2": 0.5050988499484439, + "pass@4": 0.5999355119672539, + "pass@8": 0.6770325751872642, + "pass@16": 0.7367422561935473, + "pass@32": 0.7810862819828637, + "pass@64": 0.8162268287403831, + "pass@128": 0.8511904761904762 + }, + "20": { + "pass@1": 0.4049869011976048, + "pass@2": 0.5131853664717809, + "pass@4": 0.6042743593240965, + "pass@8": 0.6785910123757385, + "pass@16": 0.738234545184363, + "pass@32": 0.7842155855252223, + "pass@64": 0.8183162239457017, + "pass@128": 0.8413173652694611 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 666988, + "answer_accuracy": 0.8325361479469564, + "avg_response_len": 308.34811870905895, + "resp_tokens_sum": 247033712, + "pass_at_k": { + "pass@1": 0.8325361479469564, + "pass@2": 0.8700066196016242, + "pass@4": 0.8969526549634996, + "pass@8": 0.9167296000966144, + "pass@16": 0.9314247396654343, + "pass@32": 0.9424912315875398, + "pass@64": 0.951062834467024, + "pass@128": 0.9579805080683815 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9673334851333878, + "pass@4": 0.975943412577159, + "pass@8": 0.9832993500745408, + "pass@16": 0.9887067340999656, + "pass@32": 0.9917497873815814, + "pass@64": 0.9932677279113761, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985753676470588, + "pass@2": 0.9996960398332562, + "pass@4": 0.9999747452524317, + "pass@8": 0.9999998417920103, + "pass@16": 0.9999999999980195, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974122634069401, + "pass@2": 0.9991131641372115, + "pass@4": 0.999854572299125, + "pass@8": 0.9999950840388245, + "pass@16": 0.9999999962921804, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978243670886076, + "pass@2": 0.9987377591448221, + "pass@4": 0.9995064679415073, + "pass@8": 0.999928806446341, + "pass@16": 0.9999988610780643, + "pass@32": 0.9999999999224412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920232732732732, + "pass@2": 0.9956769022959969, + "pass@4": 0.9981618989630802, + "pass@8": 0.9995654428829316, + "pass@16": 0.9999754906144646, + "pass@32": 0.9999999648002483, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916097266881029, + "pass@2": 0.9939979618705219, + "pass@4": 0.9958385552489217, + "pass@8": 0.9975977940296384, + "pass@16": 0.9990483367272565, + "pass@32": 0.999836024498138, + "pass@64": 0.9999978527259372, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9518105158730159, + "pass@2": 0.9663874046994128, + "pass@4": 0.974960809660697, + "pass@8": 0.9805869955097556, + "pass@16": 0.9851927553230163, + "pass@32": 0.98910883729019, + "pass@64": 0.9917724625682862, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8984852828746177, + "pass@2": 0.9240643586409498, + "pass@4": 0.941068672608584, + "pass@8": 0.9547484308868067, + "pass@16": 0.964354080653309, + "pass@32": 0.9705027070109506, + "pass@64": 0.9749939794561082, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7964891975308642, + "pass@2": 0.8458115370127349, + "pass@4": 0.8753133372795989, + "pass@8": 0.8949344772446929, + "pass@16": 0.9116664506949259, + "pass@32": 0.9266813162011269, + "pass@64": 0.9371907688348377, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.8233725890207715, + "pass@2": 0.864813941330405, + "pass@4": 0.8884574113881165, + "pass@8": 0.9057512078427568, + "pass@16": 0.9210475746514785, + "pass@32": 0.9350615129237696, + "pass@64": 0.9456666949723773, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7960069444444444, + "pass@2": 0.8479114264362789, + "pass@4": 0.8798735259365724, + "pass@8": 0.9012219064707329, + "pass@16": 0.9154730765108868, + "pass@32": 0.9257078030331487, + "pass@64": 0.9355473867528037, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6688790560471977, + "pass@2": 0.761060132278819, + "pass@4": 0.8178771328915742, + "pass@8": 0.8558947560594331, + "pass@16": 0.8849558857513954, + "pass@32": 0.9077941329129988, + "pass@64": 0.9250881004375349, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.5774896460843374, + "pass@2": 0.6752631836400717, + "pass@4": 0.7429351037445621, + "pass@8": 0.7866756488041113, + "pass@16": 0.8174313143142055, + "pass@32": 0.8439984872357021, + "pass@64": 0.8656491607233635, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.570866935483871, + "pass@2": 0.6655289560579124, + "pass@4": 0.7339587947675894, + "pass@8": 0.7841472664969731, + "pass@16": 0.8203472667980849, + "pass@32": 0.8469237045905657, + "pass@64": 0.8695213616035634, + "pass@128": 0.8903225806451613 + }, + "19": { + "pass@1": 0.3884063444108761, + "pass@2": 0.5001635463996004, + "pass@4": 0.5980781865068977, + "pass@8": 0.6776722630837357, + "pass@16": 0.7378864000697488, + "pass@32": 0.782019320918216, + "pass@64": 0.81778520074431, + "pass@128": 0.851963746223565 + }, + "20": { + "pass@1": 0.4043731231231231, + "pass@2": 0.5181975774987587, + "pass@4": 0.6152067675099175, + "pass@8": 0.6916543914883966, + "pass@16": 0.7473698050927324, + "pass@32": 0.7847132791823201, + "pass@64": 0.8109043643297321, + "pass@128": 0.8288288288288288 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3e24d5511d08d90c92b1cf099bfcddc0c0484f66 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9556939e6455676e3b4a06ad4eb40a6da9633b4a45a567557b1a21712e4921d6 +size 2710159396 diff --git a/results/composition/SDAR-100M-block8-epoch4-op17-20/summary.csv b/results/composition/SDAR-100M-block8-epoch4-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..05cd8ab980d704c503aa56197b38cbd28d03e67a --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op17-20,0.9918411458333334,0.6973953125,0.8368696546052632,0.0,0.0,0.0,199.07718576388888,374.408946875,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch4-op17-20/summary.json b/results/composition/SDAR-100M-block8-epoch4-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5a2f3d9be4b4efa638e89c33092d8005c4f8be6d --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op17-20", + "id_acc": 0.9918411458333334, + "ood_acc": 0.6973953125, + "total_acc": 0.8368696546052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.07718576388888, + "ood_avg_resp_len": 374.408946875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 678493, + "answer_accuracy": 0.8379270569870376, + "avg_response_len": 278.5486854845084, + "resp_tokens_sum": 225548670, + "pass_at_k": { + "pass@1": 0.8379270569870376, + "pass@2": 0.8753173808068597, + "pass@4": 0.9021178882634914, + "pass@8": 0.9219103027016352, + "pass@16": 0.936677817906899, + "pass@32": 0.9477679035326472, + "pass@64": 0.9563979950460396, + "pass@128": 0.9636421119190641 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.947686209439528, + "pass@2": 0.9617538992636985, + "pass@4": 0.9725950904367042, + "pass@8": 0.9805598544320808, + "pass@16": 0.9860735851683115, + "pass@32": 0.9893290501813384, + "pass@64": 0.9908876713836634, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9997566199376947, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657481, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978422619047619, + "pass@2": 0.9992364235720536, + "pass@4": 0.9998443864159838, + "pass@8": 0.9999934611094885, + "pass@16": 0.999999993102673, + "pass@32": 0.9999999999999994, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9969385785515584, + "pass@4": 0.9984315971543977, + "pass@8": 0.9995862366315358, + "pass@16": 0.9999714614533096, + "pass@32": 0.999999909391957, + "pass@64": 0.9999999999999891, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987056213017751, + "pass@2": 0.9998150887573964, + "pass@4": 0.9999939333581823, + "pass@8": 0.9999999935548729, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927634214743589, + "pass@2": 0.997842611169998, + "pass@4": 0.9996803302231451, + "pass@8": 0.9999892239856973, + "pass@16": 0.9999999910956074, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951144366197183, + "pass@2": 0.9977098813352555, + "pass@4": 0.9988829544194299, + "pass@8": 0.999601568979854, + "pass@16": 0.999951739437933, + "pass@32": 0.9999996009748804, + "pass@64": 0.9999999999993653, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9421033994334278, + "pass@2": 0.9585153827708507, + "pass@4": 0.9683495726277837, + "pass@8": 0.9750743355007809, + "pass@16": 0.9806874497365086, + "pass@32": 0.9855674217757143, + "pass@64": 0.9892681802260217, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8856594279661016, + "pass@2": 0.910248176075448, + "pass@4": 0.9253412335640251, + "pass@8": 0.9357745885481913, + "pass@16": 0.9435988842897304, + "pass@32": 0.9494690435475064, + "pass@64": 0.9538236623937969, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8312842868338558, + "pass@2": 0.8800130513662283, + "pass@4": 0.9112304953847852, + "pass@8": 0.9301201681225754, + "pass@16": 0.9426301334874778, + "pass@32": 0.9531258102406001, + "pass@64": 0.962334874645846, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.8299260124610592, + "pass@2": 0.8766005592758852, + "pass@4": 0.9035752581161002, + "pass@8": 0.9220158894644768, + "pass@16": 0.9356224843327265, + "pass@32": 0.945334584744341, + "pass@64": 0.953042570524806, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.8186867774566474, + "pass@2": 0.877645532747713, + "pass@4": 0.9140065072248914, + "pass@8": 0.937924646916376, + "pass@16": 0.9541987911233919, + "pass@32": 0.965099381921458, + "pass@64": 0.9726851787993989, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6638860887096775, + "pass@2": 0.7459883794767583, + "pass@4": 0.8018100526748667, + "pass@8": 0.8433015429764784, + "pass@16": 0.8753356035074012, + "pass@32": 0.9009404640966499, + "pass@64": 0.9223832471129497, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6308932442196532, + "pass@2": 0.7216910216876794, + "pass@4": 0.7815059672815464, + "pass@8": 0.8225626990224884, + "pass@16": 0.8534324527753754, + "pass@32": 0.8779455950894322, + "pass@64": 0.8973389717015748, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.551875, + "pass@2": 0.6496741422947138, + "pass@4": 0.7255719232417374, + "pass@8": 0.7824379510059899, + "pass@16": 0.823303427941537, + "pass@32": 0.8522402739187724, + "pass@64": 0.8735607870746009, + "pass@128": 0.8914285714285715 + }, + "19": { + "pass@1": 0.4078688063063063, + "pass@2": 0.5134215169894304, + "pass@4": 0.6041881234440287, + "pass@8": 0.6818408987273152, + "pass@16": 0.7434984007892304, + "pass@32": 0.7890733463330541, + "pass@64": 0.8258918288508185, + "pass@128": 0.8558558558558559 + }, + "20": { + "pass@1": 0.4509196696696697, + "pass@2": 0.5596455215057575, + "pass@4": 0.6476805069974355, + "pass@8": 0.7155572588224337, + "pass@16": 0.7669233647580539, + "pass@32": 0.8067563935871334, + "pass@64": 0.837089022951731, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689786, + "answer_accuracy": 0.8400550467653936, + "avg_response_len": 287.4098645752143, + "resp_tokens_sum": 235997988, + "pass_at_k": { + "pass@1": 0.8400550467653936, + "pass@2": 0.8763150081931377, + "pass@4": 0.9022558528634762, + "pass@8": 0.9215923694922954, + "pass@16": 0.9365406707220455, + "pass@32": 0.9482855604084754, + "pass@64": 0.9576451027840119, + "pass@128": 0.9653936087295402 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9353671395705522, + "pass@2": 0.9489883489928022, + "pass@4": 0.9602921086167909, + "pass@8": 0.970010030193041, + "pass@16": 0.9784615849176065, + "pass@32": 0.9861956753511112, + "pass@64": 0.9914655018647149, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968297101449275, + "pass@2": 0.9986281096656396, + "pass@4": 0.9996703835933552, + "pass@8": 0.9999750963168695, + "pass@16": 0.9999998721310288, + "pass@32": 0.9999999999994407, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955229226361032, + "pass@2": 0.9963235086749543, + "pass@4": 0.9971980834952937, + "pass@8": 0.9978938143484848, + "pass@16": 0.9985466574181527, + "pass@32": 0.9993380089577011, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998193641618497, + "pass@2": 0.9999946662646215, + "pass@4": 0.9999999959362016, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9978419235887768, + "pass@4": 0.9984183204212148, + "pass@8": 0.9991300714136, + "pass@16": 0.9997538434444305, + "pass@32": 0.9999857247050008, + "pass@64": 0.9999999936816814, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9949026651421564, + "pass@4": 0.9957427633484436, + "pass@8": 0.9969632965123278, + "pass@16": 0.9984388741597334, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950417, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9559723268072289, + "pass@2": 0.966372008704108, + "pass@4": 0.9713946641458974, + "pass@8": 0.9749261474893676, + "pass@16": 0.9779525234791777, + "pass@32": 0.9806901170607644, + "pass@64": 0.9830740245540501, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.895694553291536, + "pass@2": 0.9185176671685638, + "pass@4": 0.9316084925121034, + "pass@8": 0.9394861999416354, + "pass@16": 0.9443789712253488, + "pass@32": 0.9476920343833845, + "pass@64": 0.9504452941632805, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8206188725490197, + "pass@2": 0.8654555818390349, + "pass@4": 0.8914457194951465, + "pass@8": 0.91024432511722, + "pass@16": 0.9260177446913851, + "pass@32": 0.9387626126789113, + "pass@64": 0.9509378123272939, + "pass@128": 0.9635854341736695 + }, + "14": { + "pass@1": 0.8450520833333334, + "pass@2": 0.892094902610858, + "pass@4": 0.9161785703322166, + "pass@8": 0.9328964288192578, + "pass@16": 0.9477363031164306, + "pass@32": 0.960031109634046, + "pass@64": 0.9693199907235079, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8125236742424242, + "pass@2": 0.8730982313290383, + "pass@4": 0.9092206175932549, + "pass@8": 0.9321373846919218, + "pass@16": 0.947976048382885, + "pass@32": 0.9615416739411536, + "pass@64": 0.9736111224580455, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6658653846153846, + "pass@2": 0.75107889158086, + "pass@4": 0.807817280051532, + "pass@8": 0.8455472972839031, + "pass@16": 0.8726949869908102, + "pass@32": 0.8942385463597932, + "pass@64": 0.9125603746766824, + "pass@128": 0.9287749287749287 + }, + "17": { + "pass@1": 0.6382472826086957, + "pass@2": 0.7308938199980439, + "pass@4": 0.7938274872085086, + "pass@8": 0.8379544601446831, + "pass@16": 0.8705400266432815, + "pass@32": 0.8951335125041588, + "pass@64": 0.9145615683410627, + "pass@128": 0.9347826086956522 + }, + "18": { + "pass@1": 0.597219669117647, + "pass@2": 0.6891023043075495, + "pass@4": 0.7591783864333136, + "pass@8": 0.8121559273453292, + "pass@16": 0.8524030793941533, + "pass@32": 0.8851876403918829, + "pass@64": 0.9114327068575712, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.39669363839285715, + "pass@2": 0.5050988499484439, + "pass@4": 0.5999355119672539, + "pass@8": 0.6770325751872642, + "pass@16": 0.7367422561935473, + "pass@32": 0.7810862819828637, + "pass@64": 0.8162268287403831, + "pass@128": 0.8511904761904762 + }, + "20": { + "pass@1": 0.4049869011976048, + "pass@2": 0.5131853664717809, + "pass@4": 0.6042743593240965, + "pass@8": 0.6785910123757385, + "pass@16": 0.738234545184363, + "pass@32": 0.7842155855252223, + "pass@64": 0.8183162239457017, + "pass@128": 0.8413173652694611 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 666988, + "answer_accuracy": 0.8325361479469564, + "avg_response_len": 308.34811870905895, + "resp_tokens_sum": 247033712, + "pass_at_k": { + "pass@1": 0.8325361479469564, + "pass@2": 0.8700066196016242, + "pass@4": 0.8969526549634996, + "pass@8": 0.9167296000966144, + "pass@16": 0.9314247396654343, + "pass@32": 0.9424912315875398, + "pass@64": 0.951062834467024, + "pass@128": 0.9579805080683815 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9673334851333878, + "pass@4": 0.975943412577159, + "pass@8": 0.9832993500745408, + "pass@16": 0.9887067340999656, + "pass@32": 0.9917497873815814, + "pass@64": 0.9932677279113761, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985753676470588, + "pass@2": 0.9996960398332562, + "pass@4": 0.9999747452524317, + "pass@8": 0.9999998417920103, + "pass@16": 0.9999999999980195, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974122634069401, + "pass@2": 0.9991131641372115, + "pass@4": 0.999854572299125, + "pass@8": 0.9999950840388245, + "pass@16": 0.9999999962921804, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978243670886076, + "pass@2": 0.9987377591448221, + "pass@4": 0.9995064679415073, + "pass@8": 0.999928806446341, + "pass@16": 0.9999988610780643, + "pass@32": 0.9999999999224412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920232732732732, + "pass@2": 0.9956769022959969, + "pass@4": 0.9981618989630802, + "pass@8": 0.9995654428829316, + "pass@16": 0.9999754906144646, + "pass@32": 0.9999999648002483, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916097266881029, + "pass@2": 0.9939979618705219, + "pass@4": 0.9958385552489217, + "pass@8": 0.9975977940296384, + "pass@16": 0.9990483367272565, + "pass@32": 0.999836024498138, + "pass@64": 0.9999978527259372, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9518105158730159, + "pass@2": 0.9663874046994128, + "pass@4": 0.974960809660697, + "pass@8": 0.9805869955097556, + "pass@16": 0.9851927553230163, + "pass@32": 0.98910883729019, + "pass@64": 0.9917724625682862, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8984852828746177, + "pass@2": 0.9240643586409498, + "pass@4": 0.941068672608584, + "pass@8": 0.9547484308868067, + "pass@16": 0.964354080653309, + "pass@32": 0.9705027070109506, + "pass@64": 0.9749939794561082, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7964891975308642, + "pass@2": 0.8458115370127349, + "pass@4": 0.8753133372795989, + "pass@8": 0.8949344772446929, + "pass@16": 0.9116664506949259, + "pass@32": 0.9266813162011269, + "pass@64": 0.9371907688348377, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.8233725890207715, + "pass@2": 0.864813941330405, + "pass@4": 0.8884574113881165, + "pass@8": 0.9057512078427568, + "pass@16": 0.9210475746514785, + "pass@32": 0.9350615129237696, + "pass@64": 0.9456666949723773, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7960069444444444, + "pass@2": 0.8479114264362789, + "pass@4": 0.8798735259365724, + "pass@8": 0.9012219064707329, + "pass@16": 0.9154730765108868, + "pass@32": 0.9257078030331487, + "pass@64": 0.9355473867528037, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6688790560471977, + "pass@2": 0.761060132278819, + "pass@4": 0.8178771328915742, + "pass@8": 0.8558947560594331, + "pass@16": 0.8849558857513954, + "pass@32": 0.9077941329129988, + "pass@64": 0.9250881004375349, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.5774896460843374, + "pass@2": 0.6752631836400717, + "pass@4": 0.7429351037445621, + "pass@8": 0.7866756488041113, + "pass@16": 0.8174313143142055, + "pass@32": 0.8439984872357021, + "pass@64": 0.8656491607233635, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.570866935483871, + "pass@2": 0.6655289560579124, + "pass@4": 0.7339587947675894, + "pass@8": 0.7841472664969731, + "pass@16": 0.8203472667980849, + "pass@32": 0.8469237045905657, + "pass@64": 0.8695213616035634, + "pass@128": 0.8903225806451613 + }, + "19": { + "pass@1": 0.3884063444108761, + "pass@2": 0.5001635463996004, + "pass@4": 0.5980781865068977, + "pass@8": 0.6776722630837357, + "pass@16": 0.7378864000697488, + "pass@32": 0.782019320918216, + "pass@64": 0.81778520074431, + "pass@128": 0.851963746223565 + }, + "20": { + "pass@1": 0.4043731231231231, + "pass@2": 0.5181975774987587, + "pass@4": 0.6152067675099175, + "pass@8": 0.6916543914883966, + "pass@16": 0.7473698050927324, + "pass@32": 0.7847132791823201, + "pass@64": 0.8109043643297321, + "pass@128": 0.8288288288288288 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7ef97a7ee430bf93595375015e976294d37fb0ee --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c38d63e9a382d0c475d9101fc6948f7953080335c47f1b70a27afc4e571c3e8 +size 1789246734 diff --git a/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json b/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1bdf35d9592bf325daac5dbc43cb2c542d9fe90a --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op7-10", + "id": { + "answer_accuracy": 0.994, + "avg_response_len": 200.33833854166667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.994, + "pass@2": 0.9956650672572176, + "pass@4": 0.9968476424405281, + "pass@8": 0.9977425902955549, + "pass@16": 0.998360031800675, + "pass@32": 0.99881997872285, + "pass@64": 0.9992241214506254, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9576875, + "pass@2": 0.9681493602362207, + "pass@4": 0.9764409844394448, + "pass@8": 0.9830904285785195, + "pass@16": 0.987603875673519, + "pass@32": 0.9908814180722857, + "pass@64": 0.9936987618034088, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99984375, + "pass@2": 0.9999933562992126, + "pass@4": 0.9999999802212223, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990546875, + "pass@2": 0.9994500492125985, + "pass@4": 0.9997259424446944, + "pass@8": 0.9999287673442558, + "pass@16": 0.9999959202789485, + "pass@32": 0.9999999941392413, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999765625, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956171875, + "pass@2": 0.9977913385826772, + "pass@4": 0.9987899146044246, + "pass@8": 0.9990465879847814, + "pass@16": 0.9991248829432814, + "pass@32": 0.9992499999981759, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993890625, + "pass@2": 0.9956060531496063, + "pass@4": 0.9966719720659917, + "pass@8": 0.9976175287524772, + "pass@16": 0.9985156073103248, + "pass@32": 0.9992483962959446, + "pass@64": 0.9998183312522196, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9576875, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.99984375, + "6": 0.9990546875, + "7": 0.9999765625, + "8": 0.9956171875, + "9": 0.993890625 + }, + "per_op_avg_response_len": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376870, + "answer_accuracy": 0.9943589581222559, + "avg_response_len": 190.8690449805809, + "resp_tokens_sum": 72340895, + "pass_at_k": { + "pass@1": 0.9943589581222559, + "pass@2": 0.996177755041258, + "pass@4": 0.9971794507196228, + "pass@8": 0.9976981022527546, + "pass@16": 0.9981050816883029, + "pass@32": 0.9986103513984367, + "pass@64": 0.9991686752593077, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9625737463126843, + "pass@2": 0.9730252537569976, + "pass@4": 0.9790491030988384, + "pass@8": 0.9823822861395556, + "pass@16": 0.9851782276001185, + "pass@32": 0.9887806819385732, + "pass@64": 0.9929144734452152, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978246631736527, + "pass@2": 0.9984241654486302, + "pass@4": 0.9991805660332878, + "pass@8": 0.9997867287338961, + "pass@16": 0.9999877852663126, + "pass@32": 0.9999999824528186, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995292467948718, + "pass@2": 0.9984723589238845, + "pass@4": 0.9997657519372578, + "pass@8": 0.9999935417446474, + "pass@16": 0.9999999969902831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.99670414217589, + "pass@4": 0.9974577013260665, + "pass@8": 0.997830234782101, + "pass@16": 0.9983600236132614, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388220, + "answer_accuracy": 0.9937643348623854, + "avg_response_len": 198.363544909076, + "resp_tokens_sum": 77491909, + "pass_at_k": { + "pass@1": 0.9937643348623854, + "pass@2": 0.9953304467962144, + "pass@4": 0.9965529115135181, + "pass@8": 0.9976747403183903, + "pass@16": 0.99852526799435, + "pass@32": 0.9991184176716927, + "pass@64": 0.999592243195273, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9544190950920245, + "pass@2": 0.9655979783585334, + "pass@4": 0.9755531901495438, + "pass@8": 0.9849001130563421, + "pass@16": 0.9914179568700926, + "pass@32": 0.9953896244003085, + "pass@64": 0.9980909065259532, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999992867739359, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998433022922636, + "pass@2": 0.9999925969586897, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957086267605634, + "pass@2": 0.9968146972385495, + "pass@4": 0.9972236687843596, + "pass@8": 0.997358535350474, + "pass@16": 0.9975352112222411, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925383607784432, + "pass@2": 0.994316239568108, + "pass@4": 0.9953136485746168, + "pass@8": 0.9962981752872685, + "pass@16": 0.9975205508838629, + "pass@32": 0.99868979993864, + "pass@64": 0.9996344033069231, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379998, + "answer_accuracy": 0.9938849598259123, + "avg_response_len": 211.74297476565116, + "resp_tokens_sum": 80956962, + "pass_at_k": { + "pass@1": 0.9938849598259123, + "pass@2": 0.9954987442302473, + "pass@4": 0.9968198669048192, + "pass@8": 0.9978560175553608, + "pass@16": 0.9984439308363754, + "pass@32": 0.9987228481020659, + "pass@64": 0.998902952588164, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559235074626866, + "pass@2": 0.9656980843812436, + "pass@4": 0.9746656671647387, + "pass@8": 0.9820459603607242, + "pass@16": 0.9863468733654595, + "pass@32": 0.9886201472854002, + "pass@64": 0.9902182679940894, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999815452755905, + "pass@4": 0.9999999418271246, + "pass@8": 0.9999999999999074, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999482452681388, + "pass@2": 0.9999336328522814, + "pass@4": 0.9999988538498933, + "pass@8": 0.9999999998332318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995823948948949, + "pass@2": 0.9981944296658863, + "pass@4": 0.9995453380827396, + "pass@8": 0.9999589276607602, + "pass@16": 0.9999996513465985, + "pass@32": 0.9999999999945224, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941217845659164, + "pass@2": 0.9957378092006989, + "pass@4": 0.997233856820952, + "pass@8": 0.998791655494803, + "pass@16": 0.9997618486572567, + "pass@32": 0.9999916332542949, + "pass@64": 0.9999999987229116, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.48264140625, + "avg_response_len": 336.96297421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.48264140625, + "pass@2": 0.54950445374015, + "pass@4": 0.6049536407011535, + "pass@8": 0.6506849035948025, + "pass@16": 0.6886568937359366, + "pass@32": 0.7208077039860731, + "pass@64": 0.749042282743256, + "pass@128": 0.7745 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9440390625, + "pass@2": 0.9602047244094495, + "pass@4": 0.9703773471128609, + "pass@8": 0.9773500577446682, + "pass@16": 0.9816367907916462, + "pass@32": 0.9844934582399582, + "pass@64": 0.9866168302420555, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8583203125, + "pass@2": 0.896563607283465, + "pass@4": 0.917817725534308, + "pass@8": 0.9320315514858477, + "pass@16": 0.9425378082351825, + "pass@32": 0.9506772811040405, + "pass@64": 0.9573200207648929, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.7090390625, + "pass@2": 0.7841931594488197, + "pass@4": 0.8329807440007498, + "pass@8": 0.8652811245279454, + "pass@16": 0.8880327956833413, + "pass@32": 0.9057864037962902, + "pass@64": 0.9204816066732862, + "pass@128": 0.933 + }, + "14": { + "pass@1": 0.6603203125, + "pass@2": 0.7414966781496067, + "pass@4": 0.7985705101237346, + "pass@8": 0.8371451068534077, + "pass@16": 0.8642135734222964, + "pass@32": 0.8856339761559466, + "pass@64": 0.9033478894497897, + "pass@128": 0.917 + }, + "15": { + "pass@1": 0.5500703125, + "pass@2": 0.6516924212598431, + "pass@4": 0.7275813644544425, + "pass@8": 0.7824487694055557, + "pass@16": 0.8224994011911744, + "pass@32": 0.8523507828527137, + "pass@64": 0.8752890364832124, + "pass@128": 0.895 + }, + "16": { + "pass@1": 0.308140625, + "pass@2": 0.3923243110236223, + "pass@4": 0.4673418410198722, + "pass@8": 0.5332682864207966, + "pass@16": 0.5906135151978855, + "pass@32": 0.6396948092170933, + "pass@64": 0.6836045161679587, + "pass@128": 0.724 + }, + "17": { + "pass@1": 0.256015625, + "pass@2": 0.33540366633858254, + "pass@4": 0.409378450881139, + "pass@8": 0.47532668396617, + "pass@16": 0.5329195441450473, + "pass@32": 0.5831595117198585, + "pass@64": 0.6292019186086034, + "pass@128": 0.673 + }, + "18": { + "pass@1": 0.21353125, + "pass@2": 0.28199815452755933, + "pass@4": 0.34949067107236614, + "pass@8": 0.41318767710351856, + "pass@16": 0.47225910889314054, + "pass@32": 0.5252039399741631, + "pass@64": 0.5728921642215775, + "pass@128": 0.619 + }, + "19": { + "pass@1": 0.1728046875, + "pass@2": 0.23523597440944896, + "pass@4": 0.29625588732658387, + "pass@8": 0.3515368692538589, + "pass@16": 0.40054989229306903, + "pass@32": 0.4450398241793384, + "pass@64": 0.48646895299928694, + "pass@128": 0.524 + }, + "20": { + "pass@1": 0.1541328125, + "pass@2": 0.21593184055118106, + "pass@4": 0.2797418654855639, + "pass@8": 0.33927290918628406, + "pass@16": 0.3913065075064811, + "pass@32": 0.43603705262132386, + "pass@64": 0.4751998918218427, + "pass@128": 0.509 + } + }, + "per_op_accuracy": { + "11": 0.9440390625, + "12": 0.8583203125, + "13": 0.7090390625, + "14": 0.6603203125, + "15": 0.5500703125, + "16": 0.308140625, + "17": 0.256015625, + "18": 0.21353125, + "19": 0.1728046875, + "20": 0.1541328125 + }, + "per_op_avg_response_len": { + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 210156, + "answer_accuracy": 0.4879179049034175, + "avg_response_len": 320.37236255572066, + "resp_tokens_sum": 137990784, + "pass_at_k": { + "pass@1": 0.4879179049034175, + "pass@2": 0.553034325970213, + "pass@4": 0.6074735046585026, + "pass@8": 0.6525358099481483, + "pass@16": 0.6896515612060651, + "pass@32": 0.7205012668036456, + "pass@64": 0.7478684166087778, + "pass@128": 0.7735512630014859 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9551942294394504, + "pass@4": 0.967026604677248, + "pass@8": 0.9754982084288127, + "pass@16": 0.9808253815147637, + "pass@32": 0.9850623210750536, + "pass@64": 0.9884474194474718, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8624426200564972, + "pass@2": 0.9017520519151211, + "pass@4": 0.9232137743904894, + "pass@8": 0.9364155828181312, + "pass@16": 0.9446959074267556, + "pass@32": 0.9501805451185891, + "pass@64": 0.9558795802602567, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.734203565830721, + "pass@2": 0.801667443166391, + "pass@4": 0.8452445452547275, + "pass@8": 0.8750841253152706, + "pass@16": 0.8976730768256509, + "pass@32": 0.916616734947578, + "pass@64": 0.932356215696857, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6615313473520249, + "pass@2": 0.741312673240611, + "pass@4": 0.7987274321434115, + "pass@8": 0.8374862263892824, + "pass@16": 0.8636665344499674, + "pass@32": 0.883744442134681, + "pass@64": 0.8996549245285982, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.5426752167630058, + "pass@2": 0.6423150829502521, + "pass@4": 0.7232531651571007, + "pass@8": 0.7860465523211205, + "pass@16": 0.8328430235526624, + "pass@32": 0.8664772705101282, + "pass@64": 0.8907010774062084, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.3149193548387097, + "pass@2": 0.39902527305054625, + "pass@4": 0.4697059656857409, + "pass@8": 0.5302840910372171, + "pass@16": 0.5838085174066465, + "pass@32": 0.6282172083334884, + "pass@64": 0.6665010235625493, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.25410946531791906, + "pass@2": 0.32993918119339133, + "pass@4": 0.40069865038546476, + "pass@8": 0.46448555649405504, + "pass@16": 0.5182130045168546, + "pass@32": 0.5623529995882736, + "pass@64": 0.6040674100873197, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.20060267857142858, + "pass@2": 0.2679288526434197, + "pass@4": 0.3344746652204189, + "pass@8": 0.39710824121270083, + "pass@16": 0.4566949168331932, + "pass@32": 0.5120978334164232, + "pass@64": 0.5656579621163826, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18132976726726727, + "pass@2": 0.2433429985103213, + "pass@4": 0.30306926977596277, + "pass@8": 0.3572313687285545, + "pass@16": 0.40418699091341775, + "pass@32": 0.44416817763081634, + "pass@64": 0.47951343317052403, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.16441441441441443, + "pass@2": 0.22810002423683523, + "pass@4": 0.29267682502525005, + "pass@8": 0.35116514670673493, + "pass@16": 0.40103786340810277, + "pass@32": 0.4445894691442985, + "pass@64": 0.48528400560602686, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 206093, + "answer_accuracy": 0.4787694209039548, + "avg_response_len": 332.3799481489741, + "resp_tokens_sum": 143077602, + "pass_at_k": { + "pass@1": 0.4787694209039548, + "pass@2": 0.5458204338669327, + "pass@4": 0.6009252818921734, + "pass@8": 0.6463440310362696, + "pass@16": 0.6849403853498071, + "pass@32": 0.7189012958637673, + "pass@64": 0.7489942281363414, + "pass@128": 0.7749033600951531 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9485363328313253, + "pass@2": 0.9614666836637891, + "pass@4": 0.9690401578191278, + "pass@8": 0.9745595660303529, + "pass@16": 0.9776023172016188, + "pass@32": 0.979402643004502, + "pass@64": 0.981192486457403, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8519298589341693, + "pass@2": 0.882412744057463, + "pass@4": 0.8987941407402442, + "pass@8": 0.9122516853738211, + "pass@16": 0.9248490185450918, + "pass@32": 0.9363708212156535, + "pass@64": 0.9462338759817461, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6947435224089635, + "pass@2": 0.777705521184852, + "pass@4": 0.8304399998844679, + "pass@8": 0.8625365131047817, + "pass@16": 0.8834984921646472, + "pass@32": 0.8995238580003326, + "pass@64": 0.9129394155831055, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6441429093567251, + "pass@2": 0.7267150302758209, + "pass@4": 0.7861907489085788, + "pass@8": 0.8273056582111475, + "pass@16": 0.8566846397040585, + "pass@32": 0.8804814970414178, + "pass@64": 0.9011153985940141, + "pass@128": 0.9181286549707602 + }, + "15": { + "pass@1": 0.5651751893939394, + "pass@2": 0.6739184413027917, + "pass@4": 0.7489947995136972, + "pass@8": 0.7974960489060056, + "pass@16": 0.8302977072796687, + "pass@32": 0.8556934809010615, + "pass@64": 0.876922393623797, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.3212250712250712, + "pass@2": 0.3989094712519908, + "pass@4": 0.46772314010962307, + "pass@8": 0.5297939095149375, + "pass@16": 0.5862086467853515, + "pass@32": 0.6380627166781673, + "pass@64": 0.6854883475998351, + "pass@128": 0.7264957264957265 + }, + "17": { + "pass@1": 0.2698466614906832, + "pass@2": 0.35929246955543587, + "pass@4": 0.4414016571880376, + "pass@8": 0.5100927321509832, + "pass@16": 0.5673176419200605, + "pass@32": 0.6157075990502674, + "pass@64": 0.6567645873486623, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.20048253676470587, + "pass@2": 0.2711053294349235, + "pass@4": 0.34233683252828667, + "pass@8": 0.4105063409952905, + "pass@16": 0.4747878989634802, + "pass@32": 0.5336599706135036, + "pass@64": 0.5865359016158412, + "pass@128": 0.6352941176470588 + }, + "19": { + "pass@1": 0.15308779761904762, + "pass@2": 0.20727072201912244, + "pass@4": 0.2626874246745942, + "pass@8": 0.3177729661617527, + "pass@16": 0.3739255500583787, + "pass@32": 0.42984186349128706, + "pass@64": 0.482475527616294, + "pass@128": 0.5238095238095238 + }, + "20": { + "pass@1": 0.14775916916167664, + "pass@2": 0.20598456138903293, + "pass@4": 0.26647546779332226, + "pass@8": 0.32486153243092897, + "pass@16": 0.37714961566942967, + "pass@32": 0.4223905208474426, + "pass@64": 0.46155466688637004, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 201532, + "answer_accuracy": 0.48119460574572126, + "avg_response_len": 358.7356285337714, + "resp_tokens_sum": 150244221, + "pass_at_k": { + "pass@1": 0.48119460574572126, + "pass@2": 0.5496607308732678, + "pass@4": 0.6065025491541572, + "pass@8": 0.6532429887217562, + "pass@16": 0.6914538257852948, + "pass@32": 0.723082279668885, + "pass@64": 0.7502989047437116, + "pass@128": 0.7750611246943765 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9454117063492063, + "pass@2": 0.9644895950506187, + "pass@4": 0.9755416611614021, + "pass@8": 0.9823663944356187, + "pass@16": 0.9867982914476112, + "pass@32": 0.9892215283268877, + "pass@64": 0.9902824942007619, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8600917431192661, + "pass@2": 0.9047514237039184, + "pass@4": 0.930534307351489, + "pass@8": 0.9465814909296013, + "pass@16": 0.9574575537927447, + "pass@32": 0.9651714867408759, + "pass@64": 0.9696943208397095, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7000144675925926, + "pass@2": 0.7741369568387282, + "pass@4": 0.8237057225485699, + "pass@8": 0.8586535597961961, + "pass@16": 0.8835374150406778, + "pass@32": 0.9020236050675741, + "pass@64": 0.9171006250087037, + "pass@128": 0.9320987654320988 + }, + "14": { + "pass@1": 0.6755841988130564, + "pass@2": 0.7566729070772676, + "pass@4": 0.8109844756052377, + "pass@8": 0.846805617430967, + "pass@16": 0.8723752791841834, + "pass@32": 0.8926627247553403, + "pass@64": 0.9091311345903748, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5425829475308642, + "pass@2": 0.639068879410907, + "pass@4": 0.7103934736745863, + "pass@8": 0.7632807597637843, + "pass@16": 0.803510714937235, + "pass@32": 0.8338604893791952, + "pass@64": 0.8571668944592924, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.2883941740412979, + "pass@2": 0.379378324390867, + "pass@4": 0.4647851607044694, + "pass@8": 0.5395945603525556, + "pass@16": 0.6013971674931184, + "pass@32": 0.6518804161642361, + "pass@64": 0.6972943624071537, + "pass@128": 0.7404129793510325 + }, + "17": { + "pass@1": 0.24458772590361447, + "pass@2": 0.3179293206289724, + "pass@4": 0.3873656151603038, + "pass@8": 0.4529060893572603, + "pass@16": 0.5148842285661349, + "pass@32": 0.5732756836389475, + "pass@64": 0.6286638481690365, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.24243951612903225, + "pass@2": 0.30982981965963907, + "pass@4": 0.37429037156645767, + "pass@8": 0.4342826991634681, + "pass@16": 0.4870580721094836, + "pass@32": 0.5307268008704006, + "pass@64": 0.5660957126821192, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.1842428247734139, + "pass@2": 0.25546765349572986, + "pass@4": 0.3234768755302869, + "pass@8": 0.3800818937670737, + "pass@16": 0.42391733987699637, + "pass@32": 0.4613442746078676, + "pass@64": 0.49752024916745513, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.150243993993994, + "pass@2": 0.2137408077368706, + "pass@4": 0.2801131426702293, + "pass@8": 0.3418353258288631, + "pass@16": 0.3957745566306108, + "pass@32": 0.44117214844812946, + "pass@64": 0.4788019796245884, + "pass@128": 0.5105105105105106 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7248638980263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7248638980263158, + "pass@2": 0.7608436917219054, + "pass@4": 0.7905876415250559, + "pass@8": 0.8150806499267463, + "pass@16": 0.8353583801876622, + "pass@32": 0.8524977288614013, + "pass@64": 0.8675494694993787, + "pass@128": 0.8811052631578947 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9576875, + "pass@2": 0.9681493602362207, + "pass@4": 0.9764409844394448, + "pass@8": 0.9830904285785195, + "pass@16": 0.987603875673519, + "pass@32": 0.9908814180722857, + "pass@64": 0.9936987618034088, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99984375, + "pass@2": 0.9999933562992126, + "pass@4": 0.9999999802212223, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990546875, + "pass@2": 0.9994500492125985, + "pass@4": 0.9997259424446944, + "pass@8": 0.9999287673442558, + "pass@16": 0.9999959202789485, + "pass@32": 0.9999999941392413, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999765625, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956171875, + "pass@2": 0.9977913385826772, + "pass@4": 0.9987899146044246, + "pass@8": 0.9990465879847814, + "pass@16": 0.9991248829432814, + "pass@32": 0.9992499999981759, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993890625, + "pass@2": 0.9956060531496063, + "pass@4": 0.9966719720659917, + "pass@8": 0.9976175287524772, + "pass@16": 0.9985156073103248, + "pass@32": 0.9992483962959446, + "pass@64": 0.9998183312522196, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9440390625, + "pass@2": 0.9602047244094495, + "pass@4": 0.9703773471128609, + "pass@8": 0.9773500577446682, + "pass@16": 0.9816367907916462, + "pass@32": 0.9844934582399582, + "pass@64": 0.9866168302420555, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8583203125, + "pass@2": 0.896563607283465, + "pass@4": 0.917817725534308, + "pass@8": 0.9320315514858477, + "pass@16": 0.9425378082351825, + "pass@32": 0.9506772811040405, + "pass@64": 0.9573200207648929, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.7090390625, + "pass@2": 0.7841931594488197, + "pass@4": 0.8329807440007498, + "pass@8": 0.8652811245279454, + "pass@16": 0.8880327956833413, + "pass@32": 0.9057864037962902, + "pass@64": 0.9204816066732862, + "pass@128": 0.933 + }, + "14": { + "pass@1": 0.6603203125, + "pass@2": 0.7414966781496067, + "pass@4": 0.7985705101237346, + "pass@8": 0.8371451068534077, + "pass@16": 0.8642135734222964, + "pass@32": 0.8856339761559466, + "pass@64": 0.9033478894497897, + "pass@128": 0.917 + }, + "15": { + "pass@1": 0.5500703125, + "pass@2": 0.6516924212598431, + "pass@4": 0.7275813644544425, + "pass@8": 0.7824487694055557, + "pass@16": 0.8224994011911744, + "pass@32": 0.8523507828527137, + "pass@64": 0.8752890364832124, + "pass@128": 0.895 + }, + "16": { + "pass@1": 0.308140625, + "pass@2": 0.3923243110236223, + "pass@4": 0.4673418410198722, + "pass@8": 0.5332682864207966, + "pass@16": 0.5906135151978855, + "pass@32": 0.6396948092170933, + "pass@64": 0.6836045161679587, + "pass@128": 0.724 + }, + "17": { + "pass@1": 0.256015625, + "pass@2": 0.33540366633858254, + "pass@4": 0.409378450881139, + "pass@8": 0.47532668396617, + "pass@16": 0.5329195441450473, + "pass@32": 0.5831595117198585, + "pass@64": 0.6292019186086034, + "pass@128": 0.673 + }, + "18": { + "pass@1": 0.21353125, + "pass@2": 0.28199815452755933, + "pass@4": 0.34949067107236614, + "pass@8": 0.41318767710351856, + "pass@16": 0.47225910889314054, + "pass@32": 0.5252039399741631, + "pass@64": 0.5728921642215775, + "pass@128": 0.619 + }, + "19": { + "pass@1": 0.1728046875, + "pass@2": 0.23523597440944896, + "pass@4": 0.29625588732658387, + "pass@8": 0.3515368692538589, + "pass@16": 0.40054989229306903, + "pass@32": 0.4450398241793384, + "pass@64": 0.48646895299928694, + "pass@128": 0.524 + }, + "20": { + "pass@1": 0.1541328125, + "pass@2": 0.21593184055118106, + "pass@4": 0.2797418654855639, + "pass@8": 0.33927290918628406, + "pass@16": 0.3913065075064811, + "pass@32": 0.43603705262132386, + "pass@64": 0.4751998918218427, + "pass@128": 0.509 + } + }, + "per_op_accuracy": { + "10": 0.9576875, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.99984375, + "6": 0.9990546875, + "7": 0.9999765625, + "8": 0.9956171875, + "9": 0.993890625, + "11": 0.9440390625, + "12": 0.8583203125, + "13": 0.7090390625, + "14": 0.6603203125, + "15": 0.5500703125, + "16": 0.308140625, + "17": 0.256015625, + "18": 0.21353125, + "19": 0.1728046875, + "20": 0.1541328125 + }, + "per_op_avg_response_len": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125, + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125, + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 587026, + "answer_accuracy": 0.7249669024660133, + "avg_response_len": 259.7559662009959, + "resp_tokens_sum": 210331679, + "pass_at_k": { + "pass@1": 0.7249669024660133, + "pass@2": 0.7604557128622972, + "pass@4": 0.789882500277682, + "pass@8": 0.8140953337410525, + "pass@16": 0.8340288729588218, + "pass@32": 0.85067531035173, + "pass@64": 0.8654940988509915, + "pass@128": 0.879386658235852 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9625737463126843, + "pass@2": 0.9730252537569976, + "pass@4": 0.9790491030988384, + "pass@8": 0.9823822861395556, + "pass@16": 0.9851782276001185, + "pass@32": 0.9887806819385732, + "pass@64": 0.9929144734452152, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978246631736527, + "pass@2": 0.9984241654486302, + "pass@4": 0.9991805660332878, + "pass@8": 0.9997867287338961, + "pass@16": 0.9999877852663126, + "pass@32": 0.9999999824528186, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995292467948718, + "pass@2": 0.9984723589238845, + "pass@4": 0.9997657519372578, + "pass@8": 0.9999935417446474, + "pass@16": 0.9999999969902831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.99670414217589, + "pass@4": 0.9974577013260665, + "pass@8": 0.997830234782101, + "pass@16": 0.9983600236132614, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9551942294394504, + "pass@4": 0.967026604677248, + "pass@8": 0.9754982084288127, + "pass@16": 0.9808253815147637, + "pass@32": 0.9850623210750536, + "pass@64": 0.9884474194474718, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8624426200564972, + "pass@2": 0.9017520519151211, + "pass@4": 0.9232137743904894, + "pass@8": 0.9364155828181312, + "pass@16": 0.9446959074267556, + "pass@32": 0.9501805451185891, + "pass@64": 0.9558795802602567, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.734203565830721, + "pass@2": 0.801667443166391, + "pass@4": 0.8452445452547275, + "pass@8": 0.8750841253152706, + "pass@16": 0.8976730768256509, + "pass@32": 0.916616734947578, + "pass@64": 0.932356215696857, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6615313473520249, + "pass@2": 0.741312673240611, + "pass@4": 0.7987274321434115, + "pass@8": 0.8374862263892824, + "pass@16": 0.8636665344499674, + "pass@32": 0.883744442134681, + "pass@64": 0.8996549245285982, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.5426752167630058, + "pass@2": 0.6423150829502521, + "pass@4": 0.7232531651571007, + "pass@8": 0.7860465523211205, + "pass@16": 0.8328430235526624, + "pass@32": 0.8664772705101282, + "pass@64": 0.8907010774062084, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.3149193548387097, + "pass@2": 0.39902527305054625, + "pass@4": 0.4697059656857409, + "pass@8": 0.5302840910372171, + "pass@16": 0.5838085174066465, + "pass@32": 0.6282172083334884, + "pass@64": 0.6665010235625493, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.25410946531791906, + "pass@2": 0.32993918119339133, + "pass@4": 0.40069865038546476, + "pass@8": 0.46448555649405504, + "pass@16": 0.5182130045168546, + "pass@32": 0.5623529995882736, + "pass@64": 0.6040674100873197, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.20060267857142858, + "pass@2": 0.2679288526434197, + "pass@4": 0.3344746652204189, + "pass@8": 0.39710824121270083, + "pass@16": 0.4566949168331932, + "pass@32": 0.5120978334164232, + "pass@64": 0.5656579621163826, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18132976726726727, + "pass@2": 0.2433429985103213, + "pass@4": 0.30306926977596277, + "pass@8": 0.3572313687285545, + "pass@16": 0.40418699091341775, + "pass@32": 0.44416817763081634, + "pass@64": 0.47951343317052403, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.16441441441441443, + "pass@2": 0.22810002423683523, + "pass@4": 0.29267682502525005, + "pass@8": 0.35116514670673493, + "pass@16": 0.40103786340810277, + "pass@32": 0.4445894691442985, + "pass@64": 0.48528400560602686, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 594313, + "answer_accuracy": 0.7237833690568979, + "avg_response_len": 268.6203125, + "resp_tokens_sum": 220569511, + "pass_at_k": { + "pass@1": 0.7237833690568979, + "pass@2": 0.7596792895894745, + "pass@4": 0.7891490582919107, + "pass@8": 0.8134931073775019, + "pass@16": 0.8341315095635516, + "pass@32": 0.8522173762624881, + "pass@64": 0.8682187241550315, + "pass@128": 0.8819953234606391 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9544190950920245, + "pass@2": 0.9655979783585334, + "pass@4": 0.9755531901495438, + "pass@8": 0.9849001130563421, + "pass@16": 0.9914179568700926, + "pass@32": 0.9953896244003085, + "pass@64": 0.9980909065259532, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999992867739359, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998433022922636, + "pass@2": 0.9999925969586897, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957086267605634, + "pass@2": 0.9968146972385495, + "pass@4": 0.9972236687843596, + "pass@8": 0.997358535350474, + "pass@16": 0.9975352112222411, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925383607784432, + "pass@2": 0.994316239568108, + "pass@4": 0.9953136485746168, + "pass@8": 0.9962981752872685, + "pass@16": 0.9975205508838629, + "pass@32": 0.99868979993864, + "pass@64": 0.9996344033069231, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9485363328313253, + "pass@2": 0.9614666836637891, + "pass@4": 0.9690401578191278, + "pass@8": 0.9745595660303529, + "pass@16": 0.9776023172016188, + "pass@32": 0.979402643004502, + "pass@64": 0.981192486457403, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8519298589341693, + "pass@2": 0.882412744057463, + "pass@4": 0.8987941407402442, + "pass@8": 0.9122516853738211, + "pass@16": 0.9248490185450918, + "pass@32": 0.9363708212156535, + "pass@64": 0.9462338759817461, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6947435224089635, + "pass@2": 0.777705521184852, + "pass@4": 0.8304399998844679, + "pass@8": 0.8625365131047817, + "pass@16": 0.8834984921646472, + "pass@32": 0.8995238580003326, + "pass@64": 0.9129394155831055, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6441429093567251, + "pass@2": 0.7267150302758209, + "pass@4": 0.7861907489085788, + "pass@8": 0.8273056582111475, + "pass@16": 0.8566846397040585, + "pass@32": 0.8804814970414178, + "pass@64": 0.9011153985940141, + "pass@128": 0.9181286549707602 + }, + "15": { + "pass@1": 0.5651751893939394, + "pass@2": 0.6739184413027917, + "pass@4": 0.7489947995136972, + "pass@8": 0.7974960489060056, + "pass@16": 0.8302977072796687, + "pass@32": 0.8556934809010615, + "pass@64": 0.876922393623797, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.3212250712250712, + "pass@2": 0.3989094712519908, + "pass@4": 0.46772314010962307, + "pass@8": 0.5297939095149375, + "pass@16": 0.5862086467853515, + "pass@32": 0.6380627166781673, + "pass@64": 0.6854883475998351, + "pass@128": 0.7264957264957265 + }, + "17": { + "pass@1": 0.2698466614906832, + "pass@2": 0.35929246955543587, + "pass@4": 0.4414016571880376, + "pass@8": 0.5100927321509832, + "pass@16": 0.5673176419200605, + "pass@32": 0.6157075990502674, + "pass@64": 0.6567645873486623, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.20048253676470587, + "pass@2": 0.2711053294349235, + "pass@4": 0.34233683252828667, + "pass@8": 0.4105063409952905, + "pass@16": 0.4747878989634802, + "pass@32": 0.5336599706135036, + "pass@64": 0.5865359016158412, + "pass@128": 0.6352941176470588 + }, + "19": { + "pass@1": 0.15308779761904762, + "pass@2": 0.20727072201912244, + "pass@4": 0.2626874246745942, + "pass@8": 0.3177729661617527, + "pass@16": 0.3739255500583787, + "pass@32": 0.42984186349128706, + "pass@64": 0.482475527616294, + "pass@128": 0.5238095238095238 + }, + "20": { + "pass@1": 0.14775916916167664, + "pass@2": 0.20598456138903293, + "pass@4": 0.26647546779332226, + "pass@8": 0.32486153243092897, + "pass@16": 0.37714961566942967, + "pass@32": 0.4223905208474426, + "pass@64": 0.46155466688637004, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581530, + "answer_accuracy": 0.725867251158332, + "avg_response_len": 288.5859150323534, + "resp_tokens_sum": 231201183, + "pass_at_k": { + "pass@1": 0.725867251158332, + "pass@2": 0.7624292475528114, + "pass@4": 0.792774769656026, + "pass@8": 0.8177036241468999, + "pass@16": 0.8379595685217712, + "pass@32": 0.8546269957433247, + "pass@64": 0.8689409068065663, + "pass@128": 0.8819300207700911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559235074626866, + "pass@2": 0.9656980843812436, + "pass@4": 0.9746656671647387, + "pass@8": 0.9820459603607242, + "pass@16": 0.9863468733654595, + "pass@32": 0.9886201472854002, + "pass@64": 0.9902182679940894, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999815452755905, + "pass@4": 0.9999999418271246, + "pass@8": 0.9999999999999074, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999482452681388, + "pass@2": 0.9999336328522814, + "pass@4": 0.9999988538498933, + "pass@8": 0.9999999998332318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995823948948949, + "pass@2": 0.9981944296658863, + "pass@4": 0.9995453380827396, + "pass@8": 0.9999589276607602, + "pass@16": 0.9999996513465985, + "pass@32": 0.9999999999945224, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941217845659164, + "pass@2": 0.9957378092006989, + "pass@4": 0.997233856820952, + "pass@8": 0.998791655494803, + "pass@16": 0.9997618486572567, + "pass@32": 0.9999916332542949, + "pass@64": 0.9999999987229116, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9454117063492063, + "pass@2": 0.9644895950506187, + "pass@4": 0.9755416611614021, + "pass@8": 0.9823663944356187, + "pass@16": 0.9867982914476112, + "pass@32": 0.9892215283268877, + "pass@64": 0.9902824942007619, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8600917431192661, + "pass@2": 0.9047514237039184, + "pass@4": 0.930534307351489, + "pass@8": 0.9465814909296013, + "pass@16": 0.9574575537927447, + "pass@32": 0.9651714867408759, + "pass@64": 0.9696943208397095, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7000144675925926, + "pass@2": 0.7741369568387282, + "pass@4": 0.8237057225485699, + "pass@8": 0.8586535597961961, + "pass@16": 0.8835374150406778, + "pass@32": 0.9020236050675741, + "pass@64": 0.9171006250087037, + "pass@128": 0.9320987654320988 + }, + "14": { + "pass@1": 0.6755841988130564, + "pass@2": 0.7566729070772676, + "pass@4": 0.8109844756052377, + "pass@8": 0.846805617430967, + "pass@16": 0.8723752791841834, + "pass@32": 0.8926627247553403, + "pass@64": 0.9091311345903748, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5425829475308642, + "pass@2": 0.639068879410907, + "pass@4": 0.7103934736745863, + "pass@8": 0.7632807597637843, + "pass@16": 0.803510714937235, + "pass@32": 0.8338604893791952, + "pass@64": 0.8571668944592924, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.2883941740412979, + "pass@2": 0.379378324390867, + "pass@4": 0.4647851607044694, + "pass@8": 0.5395945603525556, + "pass@16": 0.6013971674931184, + "pass@32": 0.6518804161642361, + "pass@64": 0.6972943624071537, + "pass@128": 0.7404129793510325 + }, + "17": { + "pass@1": 0.24458772590361447, + "pass@2": 0.3179293206289724, + "pass@4": 0.3873656151603038, + "pass@8": 0.4529060893572603, + "pass@16": 0.5148842285661349, + "pass@32": 0.5732756836389475, + "pass@64": 0.6286638481690365, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.24243951612903225, + "pass@2": 0.30982981965963907, + "pass@4": 0.37429037156645767, + "pass@8": 0.4342826991634681, + "pass@16": 0.4870580721094836, + "pass@32": 0.5307268008704006, + "pass@64": 0.5660957126821192, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.1842428247734139, + "pass@2": 0.25546765349572986, + "pass@4": 0.3234768755302869, + "pass@8": 0.3800818937670737, + "pass@16": 0.42391733987699637, + "pass@32": 0.4613442746078676, + "pass@64": 0.49752024916745513, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.150243993993994, + "pass@2": 0.2137408077368706, + "pass@4": 0.2801131426702293, + "pass@8": 0.3418353258288631, + "pass@16": 0.3957745566306108, + "pass@32": 0.44117214844812946, + "pass@64": 0.4788019796245884, + "pass@128": 0.5105105105105106 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..70d64104c4bf747f13d648396971eb3179999db5 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf860bc36b4ab9f110df464e5a94ad398295dd82cc9bbe559e555bbf69a2f1aa +size 2595852590 diff --git a/results/composition/SDAR-100M-block8-epoch4-op7-10/summary.csv b/results/composition/SDAR-100M-block8-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a3e23590f8f9519b290537b24a41d4eab72c5202 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op7-10,0.994,0.48264140625,0.7248638980263158,0.0,0.0,0.0,200.33833854166667,336.96297421875,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch4-op7-10/summary.json b/results/composition/SDAR-100M-block8-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0fa75c4ac98093ad1c4cbaa95786483f5f9195cd --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op7-10", + "id_acc": 0.994, + "ood_acc": 0.48264140625, + "total_acc": 0.7248638980263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.33833854166667, + "ood_avg_resp_len": 336.96297421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 587026, + "answer_accuracy": 0.7249669024660133, + "avg_response_len": 259.7559662009959, + "resp_tokens_sum": 210331679, + "pass_at_k": { + "pass@1": 0.7249669024660133, + "pass@2": 0.7604557128622972, + "pass@4": 0.789882500277682, + "pass@8": 0.8140953337410525, + "pass@16": 0.8340288729588218, + "pass@32": 0.85067531035173, + "pass@64": 0.8654940988509915, + "pass@128": 0.879386658235852 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9625737463126843, + "pass@2": 0.9730252537569976, + "pass@4": 0.9790491030988384, + "pass@8": 0.9823822861395556, + "pass@16": 0.9851782276001185, + "pass@32": 0.9887806819385732, + "pass@64": 0.9929144734452152, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978246631736527, + "pass@2": 0.9984241654486302, + "pass@4": 0.9991805660332878, + "pass@8": 0.9997867287338961, + "pass@16": 0.9999877852663126, + "pass@32": 0.9999999824528186, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995292467948718, + "pass@2": 0.9984723589238845, + "pass@4": 0.9997657519372578, + "pass@8": 0.9999935417446474, + "pass@16": 0.9999999969902831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.99670414217589, + "pass@4": 0.9974577013260665, + "pass@8": 0.997830234782101, + "pass@16": 0.9983600236132614, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9551942294394504, + "pass@4": 0.967026604677248, + "pass@8": 0.9754982084288127, + "pass@16": 0.9808253815147637, + "pass@32": 0.9850623210750536, + "pass@64": 0.9884474194474718, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8624426200564972, + "pass@2": 0.9017520519151211, + "pass@4": 0.9232137743904894, + "pass@8": 0.9364155828181312, + "pass@16": 0.9446959074267556, + "pass@32": 0.9501805451185891, + "pass@64": 0.9558795802602567, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.734203565830721, + "pass@2": 0.801667443166391, + "pass@4": 0.8452445452547275, + "pass@8": 0.8750841253152706, + "pass@16": 0.8976730768256509, + "pass@32": 0.916616734947578, + "pass@64": 0.932356215696857, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6615313473520249, + "pass@2": 0.741312673240611, + "pass@4": 0.7987274321434115, + "pass@8": 0.8374862263892824, + "pass@16": 0.8636665344499674, + "pass@32": 0.883744442134681, + "pass@64": 0.8996549245285982, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.5426752167630058, + "pass@2": 0.6423150829502521, + "pass@4": 0.7232531651571007, + "pass@8": 0.7860465523211205, + "pass@16": 0.8328430235526624, + "pass@32": 0.8664772705101282, + "pass@64": 0.8907010774062084, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.3149193548387097, + "pass@2": 0.39902527305054625, + "pass@4": 0.4697059656857409, + "pass@8": 0.5302840910372171, + "pass@16": 0.5838085174066465, + "pass@32": 0.6282172083334884, + "pass@64": 0.6665010235625493, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.25410946531791906, + "pass@2": 0.32993918119339133, + "pass@4": 0.40069865038546476, + "pass@8": 0.46448555649405504, + "pass@16": 0.5182130045168546, + "pass@32": 0.5623529995882736, + "pass@64": 0.6040674100873197, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.20060267857142858, + "pass@2": 0.2679288526434197, + "pass@4": 0.3344746652204189, + "pass@8": 0.39710824121270083, + "pass@16": 0.4566949168331932, + "pass@32": 0.5120978334164232, + "pass@64": 0.5656579621163826, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18132976726726727, + "pass@2": 0.2433429985103213, + "pass@4": 0.30306926977596277, + "pass@8": 0.3572313687285545, + "pass@16": 0.40418699091341775, + "pass@32": 0.44416817763081634, + "pass@64": 0.47951343317052403, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.16441441441441443, + "pass@2": 0.22810002423683523, + "pass@4": 0.29267682502525005, + "pass@8": 0.35116514670673493, + "pass@16": 0.40103786340810277, + "pass@32": 0.4445894691442985, + "pass@64": 0.48528400560602686, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 594313, + "answer_accuracy": 0.7237833690568979, + "avg_response_len": 268.6203125, + "resp_tokens_sum": 220569511, + "pass_at_k": { + "pass@1": 0.7237833690568979, + "pass@2": 0.7596792895894745, + "pass@4": 0.7891490582919107, + "pass@8": 0.8134931073775019, + "pass@16": 0.8341315095635516, + "pass@32": 0.8522173762624881, + "pass@64": 0.8682187241550315, + "pass@128": 0.8819953234606391 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9544190950920245, + "pass@2": 0.9655979783585334, + "pass@4": 0.9755531901495438, + "pass@8": 0.9849001130563421, + "pass@16": 0.9914179568700926, + "pass@32": 0.9953896244003085, + "pass@64": 0.9980909065259532, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999992867739359, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998433022922636, + "pass@2": 0.9999925969586897, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957086267605634, + "pass@2": 0.9968146972385495, + "pass@4": 0.9972236687843596, + "pass@8": 0.997358535350474, + "pass@16": 0.9975352112222411, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925383607784432, + "pass@2": 0.994316239568108, + "pass@4": 0.9953136485746168, + "pass@8": 0.9962981752872685, + "pass@16": 0.9975205508838629, + "pass@32": 0.99868979993864, + "pass@64": 0.9996344033069231, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9485363328313253, + "pass@2": 0.9614666836637891, + "pass@4": 0.9690401578191278, + "pass@8": 0.9745595660303529, + "pass@16": 0.9776023172016188, + "pass@32": 0.979402643004502, + "pass@64": 0.981192486457403, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8519298589341693, + "pass@2": 0.882412744057463, + "pass@4": 0.8987941407402442, + "pass@8": 0.9122516853738211, + "pass@16": 0.9248490185450918, + "pass@32": 0.9363708212156535, + "pass@64": 0.9462338759817461, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6947435224089635, + "pass@2": 0.777705521184852, + "pass@4": 0.8304399998844679, + "pass@8": 0.8625365131047817, + "pass@16": 0.8834984921646472, + "pass@32": 0.8995238580003326, + "pass@64": 0.9129394155831055, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6441429093567251, + "pass@2": 0.7267150302758209, + "pass@4": 0.7861907489085788, + "pass@8": 0.8273056582111475, + "pass@16": 0.8566846397040585, + "pass@32": 0.8804814970414178, + "pass@64": 0.9011153985940141, + "pass@128": 0.9181286549707602 + }, + "15": { + "pass@1": 0.5651751893939394, + "pass@2": 0.6739184413027917, + "pass@4": 0.7489947995136972, + "pass@8": 0.7974960489060056, + "pass@16": 0.8302977072796687, + "pass@32": 0.8556934809010615, + "pass@64": 0.876922393623797, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.3212250712250712, + "pass@2": 0.3989094712519908, + "pass@4": 0.46772314010962307, + "pass@8": 0.5297939095149375, + "pass@16": 0.5862086467853515, + "pass@32": 0.6380627166781673, + "pass@64": 0.6854883475998351, + "pass@128": 0.7264957264957265 + }, + "17": { + "pass@1": 0.2698466614906832, + "pass@2": 0.35929246955543587, + "pass@4": 0.4414016571880376, + "pass@8": 0.5100927321509832, + "pass@16": 0.5673176419200605, + "pass@32": 0.6157075990502674, + "pass@64": 0.6567645873486623, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.20048253676470587, + "pass@2": 0.2711053294349235, + "pass@4": 0.34233683252828667, + "pass@8": 0.4105063409952905, + "pass@16": 0.4747878989634802, + "pass@32": 0.5336599706135036, + "pass@64": 0.5865359016158412, + "pass@128": 0.6352941176470588 + }, + "19": { + "pass@1": 0.15308779761904762, + "pass@2": 0.20727072201912244, + "pass@4": 0.2626874246745942, + "pass@8": 0.3177729661617527, + "pass@16": 0.3739255500583787, + "pass@32": 0.42984186349128706, + "pass@64": 0.482475527616294, + "pass@128": 0.5238095238095238 + }, + "20": { + "pass@1": 0.14775916916167664, + "pass@2": 0.20598456138903293, + "pass@4": 0.26647546779332226, + "pass@8": 0.32486153243092897, + "pass@16": 0.37714961566942967, + "pass@32": 0.4223905208474426, + "pass@64": 0.46155466688637004, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581530, + "answer_accuracy": 0.725867251158332, + "avg_response_len": 288.5859150323534, + "resp_tokens_sum": 231201183, + "pass_at_k": { + "pass@1": 0.725867251158332, + "pass@2": 0.7624292475528114, + "pass@4": 0.792774769656026, + "pass@8": 0.8177036241468999, + "pass@16": 0.8379595685217712, + "pass@32": 0.8546269957433247, + "pass@64": 0.8689409068065663, + "pass@128": 0.8819300207700911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559235074626866, + "pass@2": 0.9656980843812436, + "pass@4": 0.9746656671647387, + "pass@8": 0.9820459603607242, + "pass@16": 0.9863468733654595, + "pass@32": 0.9886201472854002, + "pass@64": 0.9902182679940894, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999815452755905, + "pass@4": 0.9999999418271246, + "pass@8": 0.9999999999999074, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999482452681388, + "pass@2": 0.9999336328522814, + "pass@4": 0.9999988538498933, + "pass@8": 0.9999999998332318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995823948948949, + "pass@2": 0.9981944296658863, + "pass@4": 0.9995453380827396, + "pass@8": 0.9999589276607602, + "pass@16": 0.9999996513465985, + "pass@32": 0.9999999999945224, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941217845659164, + "pass@2": 0.9957378092006989, + "pass@4": 0.997233856820952, + "pass@8": 0.998791655494803, + "pass@16": 0.9997618486572567, + "pass@32": 0.9999916332542949, + "pass@64": 0.9999999987229116, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9454117063492063, + "pass@2": 0.9644895950506187, + "pass@4": 0.9755416611614021, + "pass@8": 0.9823663944356187, + "pass@16": 0.9867982914476112, + "pass@32": 0.9892215283268877, + "pass@64": 0.9902824942007619, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8600917431192661, + "pass@2": 0.9047514237039184, + "pass@4": 0.930534307351489, + "pass@8": 0.9465814909296013, + "pass@16": 0.9574575537927447, + "pass@32": 0.9651714867408759, + "pass@64": 0.9696943208397095, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7000144675925926, + "pass@2": 0.7741369568387282, + "pass@4": 0.8237057225485699, + "pass@8": 0.8586535597961961, + "pass@16": 0.8835374150406778, + "pass@32": 0.9020236050675741, + "pass@64": 0.9171006250087037, + "pass@128": 0.9320987654320988 + }, + "14": { + "pass@1": 0.6755841988130564, + "pass@2": 0.7566729070772676, + "pass@4": 0.8109844756052377, + "pass@8": 0.846805617430967, + "pass@16": 0.8723752791841834, + "pass@32": 0.8926627247553403, + "pass@64": 0.9091311345903748, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5425829475308642, + "pass@2": 0.639068879410907, + "pass@4": 0.7103934736745863, + "pass@8": 0.7632807597637843, + "pass@16": 0.803510714937235, + "pass@32": 0.8338604893791952, + "pass@64": 0.8571668944592924, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.2883941740412979, + "pass@2": 0.379378324390867, + "pass@4": 0.4647851607044694, + "pass@8": 0.5395945603525556, + "pass@16": 0.6013971674931184, + "pass@32": 0.6518804161642361, + "pass@64": 0.6972943624071537, + "pass@128": 0.7404129793510325 + }, + "17": { + "pass@1": 0.24458772590361447, + "pass@2": 0.3179293206289724, + "pass@4": 0.3873656151603038, + "pass@8": 0.4529060893572603, + "pass@16": 0.5148842285661349, + "pass@32": 0.5732756836389475, + "pass@64": 0.6286638481690365, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.24243951612903225, + "pass@2": 0.30982981965963907, + "pass@4": 0.37429037156645767, + "pass@8": 0.4342826991634681, + "pass@16": 0.4870580721094836, + "pass@32": 0.5307268008704006, + "pass@64": 0.5660957126821192, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.1842428247734139, + "pass@2": 0.25546765349572986, + "pass@4": 0.3234768755302869, + "pass@8": 0.3800818937670737, + "pass@16": 0.42391733987699637, + "pass@32": 0.4613442746078676, + "pass@64": 0.49752024916745513, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.150243993993994, + "pass@2": 0.2137408077368706, + "pass@4": 0.2801131426702293, + "pass@8": 0.3418353258288631, + "pass@16": 0.3957745566306108, + "pass@32": 0.44117214844812946, + "pass@64": 0.4788019796245884, + "pass@128": 0.5105105105105106 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ab103af08968983167fd8b612d9728149da13670 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0bff41d577c72cfb82325f0583ce7c5b544e95c286b0cd330c207c42a4af2fd +size 1789351705 diff --git a/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json b/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe9a7672788f66757cd12dbe59a9b96296e4b0e --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9935295138888889, + "avg_response_len": 199.78453819444445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9935295138888889, + "pass@2": 0.9953216316710405, + "pass@4": 0.9966934193121689, + "pass@8": 0.9977862240274186, + "pass@16": 0.9985941094399321, + "pass@32": 0.9991509274337316, + "pass@64": 0.9994859817348817, + "pass@128": 0.9996666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9566953125, + "pass@2": 0.9671897145669298, + "pass@4": 0.9763015637420321, + "pass@8": 0.9841177284807338, + "pass@16": 0.9898433434950407, + "pass@32": 0.9935109453775409, + "pass@64": 0.9956717768827965, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9993671875, + "pass@2": 0.9999009596456694, + "pass@4": 0.9999937616235471, + "pass@8": 0.9999999729956526, + "pass@16": 0.9999999999998621, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99846875, + "pass@2": 0.9992100147637796, + "pass@4": 0.9997477704349457, + "pass@8": 0.9999643299327841, + "pass@16": 0.9999991554454004, + "pass@32": 0.9999999998305314, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936171875, + "pass@2": 0.9961348425196852, + "pass@4": 0.9976905641169854, + "pass@8": 0.9985067082658915, + "pass@16": 0.9989825851297969, + "pass@32": 0.9993892496351895, + "pass@64": 0.9997513007685047, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9937734375, + "pass@2": 0.9954635826771654, + "pass@4": 0.9965071167041619, + "pass@8": 0.9974872765717031, + "pass@16": 0.9985219008892885, + "pass@32": 0.9994581520603246, + "pass@64": 0.9999507579626357, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9566953125, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.9993671875, + "6": 0.99846875, + "7": 0.9999140625, + "8": 0.9936171875, + "9": 0.9937734375 + }, + "per_op_avg_response_len": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376718, + "answer_accuracy": 0.9939579111786558, + "avg_response_len": 190.33322515619724, + "resp_tokens_sum": 72137815, + "pass_at_k": { + "pass@1": 0.9939579111786558, + "pass@2": 0.9960199456450922, + "pass@4": 0.9973071401472489, + "pass@8": 0.9980447346715772, + "pass@16": 0.9985467053785979, + "pass@32": 0.9989975171639501, + "pass@64": 0.9992788389631969, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9618132374631269, + "pass@2": 0.9725411121176223, + "pass@4": 0.9802102696233768, + "pass@8": 0.9849936946412662, + "pass@16": 0.9884440697041347, + "pass@32": 0.991615765648179, + "pass@64": 0.9937203955529144, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.999994232898622, + "pass@4": 0.9999999956060179, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996279761904762, + "pass@2": 0.9999808617672791, + "pass@4": 0.9999999687539058, + "pass@8": 0.9999999999999978, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985965568862275, + "pass@2": 0.9993480067424206, + "pass@4": 0.9998631435105043, + "pass@8": 0.9999946418638593, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999934480268369, + "pass@4": 0.999999995840017, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9912359775641025, + "pass@2": 0.9958788392136078, + "pass@4": 0.9986500876092409, + "pass@8": 0.9997715179680895, + "pass@16": 0.9999920381521239, + "pass@32": 0.9999999947732479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972881085726961, + "pass@4": 0.9977522703148021, + "pass@8": 0.9982272475790946, + "pass@16": 0.998920396325759, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387814, + "answer_accuracy": 0.9927250573394495, + "avg_response_len": 197.7959125163827, + "resp_tokens_sum": 77270160, + "pass_at_k": { + "pass@1": 0.9927250573394495, + "pass@2": 0.9944081540051185, + "pass@4": 0.9958760632783779, + "pass@8": 0.9973098896013131, + "pass@16": 0.9985301797085706, + "pass@32": 0.999370767579105, + "pass@64": 0.9998439578635286, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9461752300613497, + "pass@2": 0.9579337954688179, + "pass@4": 0.9694229502945566, + "pass@8": 0.9810037051895499, + "pass@16": 0.9905673067988211, + "pass@32": 0.9965809863505134, + "pass@64": 0.9993884580111478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9990942028985508, + "pass@2": 0.9997621391076116, + "pass@4": 0.9999820552322264, + "pass@8": 0.9999999217268679, + "pass@16": 0.9999999999996003, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980524713467048, + "pass@2": 0.9987414829772353, + "pass@4": 0.9994561608344803, + "pass@8": 0.9999035515654319, + "pass@16": 0.9999975847114193, + "pass@32": 0.9999999995144166, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965228873239437, + "pass@2": 0.9971952284573583, + "pass@4": 0.9973554949117276, + "pass@8": 0.997525506974028, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928190494011976, + "pass@2": 0.9945048387477016, + "pass@4": 0.9955588426633797, + "pass@8": 0.996690741627332, + "pass@16": 0.9980681831422826, + "pass@32": 0.9992670805169979, + "pass@64": 0.9999136334847853, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380014, + "answer_accuracy": 0.9939268078339472, + "avg_response_len": 211.1854834491128, + "resp_tokens_sum": 80743813, + "pass_at_k": { + "pass@1": 0.9939268078339472, + "pass@2": 0.9955627519118282, + "pass@4": 0.9969201830290966, + "pass@8": 0.9980166634820948, + "pass@16": 0.9987064217819897, + "pass@32": 0.9990783782824575, + "pass@64": 0.9993255554216345, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9617537313432836, + "pass@2": 0.970781672346927, + "pass@4": 0.9790400016229313, + "pass@8": 0.98626166598672, + "pass@16": 0.9905548114893244, + "pass@32": 0.9924411918583297, + "pass@64": 0.9940296282947009, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994025735294118, + "pass@2": 0.9999677946966188, + "pass@4": 0.9999998894439663, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987923895899053, + "pass@2": 0.999580450955066, + "pass@4": 0.999947256691494, + "pass@8": 0.9999993061637832, + "pass@16": 0.9999999999491976, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927505630630631, + "pass@2": 0.9952442600080397, + "pass@4": 0.9971487570247412, + "pass@8": 0.9983676867389416, + "pass@16": 0.9992487477653507, + "pass@32": 0.9998506753392894, + "pass@64": 0.9999979947884918, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925391881028939, + "pass@2": 0.9944105723219485, + "pass@4": 0.9961042035139499, + "pass@8": 0.997498057805773, + "pass@16": 0.9985542991129313, + "pass@32": 0.9994502950777389, + "pass@64": 0.9999555508074901, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5771015625, + "avg_response_len": 354.4255953125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5771015625, + "pass@2": 0.6480805364173114, + "pass@4": 0.7060837591488446, + "pass@8": 0.7523766467121149, + "pass@16": 0.789200237911657, + "pass@32": 0.8187643350468798, + "pass@64": 0.8428475613537451, + "pass@128": 0.8628 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.957140625, + "pass@2": 0.9682801427165355, + "pass@4": 0.9749531122047241, + "pass@8": 0.9791891767647783, + "pass@16": 0.98211903213778, + "pass@32": 0.9846358344909678, + "pass@64": 0.9867111407997167, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9026015625, + "pass@2": 0.924557701771654, + "pass@4": 0.9376451166104238, + "pass@8": 0.9468465265856568, + "pass@16": 0.954191808841156, + "pass@32": 0.9603990401824475, + "pass@64": 0.9657758332302137, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.814109375, + "pass@2": 0.8641605561023634, + "pass@4": 0.895520004874391, + "pass@8": 0.9164012838141359, + "pass@16": 0.9319405625887556, + "pass@32": 0.9444791104314104, + "pass@64": 0.9538452723690258, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7910234375, + "pass@2": 0.8541248769685043, + "pass@4": 0.8911382658417697, + "pass@8": 0.9145212717761322, + "pass@16": 0.9314696532384339, + "pass@32": 0.9443763740796397, + "pass@64": 0.9539442507806651, + "pass@128": 0.961 + }, + "15": { + "pass@1": 0.6977265625, + "pass@2": 0.782586245078741, + "pass@4": 0.8406202425946752, + "pass@8": 0.8807529357498628, + "pass@16": 0.9092865874591104, + "pass@32": 0.9296244730051035, + "pass@64": 0.9429202877342062, + "pass@128": 0.951 + }, + "16": { + "pass@1": 0.508703125, + "pass@2": 0.6256612942913391, + "pass@4": 0.7164985766779153, + "pass@8": 0.7826049747381607, + "pass@16": 0.8290852993476789, + "pass@32": 0.8612613381637253, + "pass@64": 0.8844828159208924, + "pass@128": 0.903 + }, + "17": { + "pass@1": 0.3892578125, + "pass@2": 0.4965643454724407, + "pass@4": 0.5878991002062244, + "pass@8": 0.6593557882776578, + "pass@16": 0.7147717915717123, + "pass@32": 0.7588696172178708, + "pass@64": 0.7963482777124878, + "pass@128": 0.831 + }, + "18": { + "pass@1": 0.320015625, + "pass@2": 0.4193448572834649, + "pass@4": 0.5105761571053615, + "pass@8": 0.5890740631550005, + "pass@16": 0.6547608784288393, + "pass@32": 0.7090559625862405, + "pass@64": 0.7541917451827876, + "pass@128": 0.793 + }, + "19": { + "pass@1": 0.208, + "pass@2": 0.28776230314960616, + "pass@4": 0.3691554985939253, + "pass@8": 0.44260925321184885, + "pass@16": 0.5048892048968053, + "pass@32": 0.5590981633198217, + "pass@64": 0.6067321047916664, + "pass@128": 0.645 + }, + "20": { + "pass@1": 0.1824375, + "pass@2": 0.2577630413385826, + "pass@4": 0.33683151677915263, + "pass@8": 0.41241119304797336, + "pass@16": 0.47948756060617204, + "pass@32": 0.5358434369915598, + "pass@64": 0.5835238850156994, + "pass@128": 0.626 + } + }, + "per_op_accuracy": { + "11": 0.957140625, + "12": 0.9026015625, + "13": 0.814109375, + "14": 0.7910234375, + "15": 0.6977265625, + "16": 0.508703125, + "17": 0.3892578125, + "18": 0.320015625, + "19": 0.208, + "20": 0.1824375 + }, + "per_op_avg_response_len": { + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 251749, + "answer_accuracy": 0.5844841196136701, + "avg_response_len": 336.48382940193164, + "resp_tokens_sum": 144930315, + "pass_at_k": { + "pass@1": 0.5844841196136701, + "pass@2": 0.6537676156240151, + "pass@4": 0.709439007131913, + "pass@8": 0.7540203073917721, + "pass@16": 0.7902984564424712, + "pass@32": 0.8197780068817672, + "pass@64": 0.8446094992676029, + "pass@128": 0.8665676077265974 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9514651203966006, + "pass@2": 0.964051451562535, + "pass@4": 0.9726533114840814, + "pass@8": 0.9784351301207374, + "pass@16": 0.9821156095662544, + "pass@32": 0.9847244967783608, + "pass@64": 0.9857797640615148, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8961864406779662, + "pass@2": 0.9169464416789005, + "pass@4": 0.930045741104395, + "pass@8": 0.9405154502869694, + "pass@16": 0.9496226622064528, + "pass@32": 0.9572528675630418, + "pass@64": 0.9642242554401338, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8420111677115988, + "pass@2": 0.889376943820502, + "pass@4": 0.9181533575936236, + "pass@8": 0.9363334327363076, + "pass@16": 0.949785723044496, + "pass@32": 0.9610582398326716, + "pass@64": 0.9701180751617702, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8046388239875389, + "pass@2": 0.8602070915201021, + "pass@4": 0.8914461434143157, + "pass@8": 0.9119244313646143, + "pass@16": 0.9272654211635459, + "pass@32": 0.9383072121466731, + "pass@64": 0.9462294718077346, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.7136244580924855, + "pass@2": 0.8005655892995307, + "pass@4": 0.8601507764022269, + "pass@8": 0.9013584538970695, + "pass@16": 0.9310673045046043, + "pass@32": 0.9517436419640584, + "pass@64": 0.9651808484660429, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.523109879032258, + "pass@2": 0.6348286290322582, + "pass@4": 0.7166433194842577, + "pass@8": 0.7779303950852562, + "pass@16": 0.8253214388050136, + "pass@32": 0.8612953308847185, + "pass@64": 0.8875623036766475, + "pass@128": 0.9032258064516129 + }, + "17": { + "pass@1": 0.40713059971098264, + "pass@2": 0.5174622940466982, + "pass@4": 0.6078873918758712, + "pass@8": 0.6741004742749396, + "pass@16": 0.7231938373226637, + "pass@32": 0.7607190313851854, + "pass@64": 0.7937445756323053, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.3004017857142857, + "pass@2": 0.3996291479190101, + "pass@4": 0.4907275606620602, + "pass@8": 0.5703714538130185, + "pass@16": 0.6388093892798813, + "pass@32": 0.6961804908111601, + "pass@64": 0.7459388686836753, + "pass@128": 0.7914285714285715 + }, + "19": { + "pass@1": 0.2051661036036036, + "pass@2": 0.27981980504126164, + "pass@4": 0.35245659495265824, + "pass@8": 0.4186875923466568, + "pass@16": 0.4774648463192164, + "pass@32": 0.530660247056415, + "pass@64": 0.5814726524949139, + "pass@128": 0.6306306306306306 + }, + "20": { + "pass@1": 0.19031531531531531, + "pass@2": 0.26815730592797515, + "pass@4": 0.3495825434545906, + "pass@8": 0.42705696041508395, + "pass@16": 0.4956881270636332, + "pass@32": 0.5538044007945283, + "pass@64": 0.6040435053189296, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 250403, + "answer_accuracy": 0.5817048580136782, + "avg_response_len": 349.934119926405, + "resp_tokens_sum": 150634041, + "pass_at_k": { + "pass@1": 0.5817048580136782, + "pass@2": 0.6533466322954088, + "pass@4": 0.7110509270919676, + "pass@8": 0.7567381183491475, + "pass@16": 0.7934898486662506, + "pass@32": 0.8236383292718752, + "pass@64": 0.8481969597645258, + "pass@128": 0.8673803151947665 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9585137424698795, + "pass@2": 0.9671995066881702, + "pass@4": 0.9721630024033138, + "pass@8": 0.9750416872186753, + "pass@16": 0.9774501659354414, + "pass@32": 0.9807857011053507, + "pass@64": 0.9848766720425005, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9050499608150471, + "pass@2": 0.923858545405179, + "pass@4": 0.9327921901611824, + "pass@8": 0.9387195559136081, + "pass@16": 0.9449860445424791, + "pass@32": 0.952132953833246, + "pass@64": 0.9586975480454887, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8070290616246498, + "pass@2": 0.8562368352852956, + "pass@4": 0.8857674544883569, + "pass@8": 0.9058580167888862, + "pass@16": 0.9223591986829233, + "pass@32": 0.9360243594008619, + "pass@64": 0.9450110575563182, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7976973684210527, + "pass@2": 0.867424569461712, + "pass@4": 0.9068931860491114, + "pass@8": 0.9283267548205743, + "pass@16": 0.9418403403371376, + "pass@32": 0.9524016855817911, + "pass@64": 0.9614012630596699, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6974431818181818, + "pass@2": 0.7863684830589354, + "pass@4": 0.8486680735930725, + "pass@8": 0.891768548521778, + "pass@16": 0.9207707356638722, + "pass@32": 0.9400995306433965, + "pass@64": 0.9521569927151597, + "pass@128": 0.9606060606060606 + }, + "16": { + "pass@1": 0.5107505341880342, + "pass@2": 0.6242663677456984, + "pass@4": 0.7127050812024565, + "pass@8": 0.7776122661212889, + "pass@16": 0.8247779610106144, + "pass@32": 0.8566217836455938, + "pass@64": 0.8764856255133433, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4086034549689441, + "pass@2": 0.5166432575683475, + "pass@4": 0.6060352741233433, + "pass@8": 0.6743064342428341, + "pass@16": 0.7267822114247295, + "pass@32": 0.7705926178718194, + "pass@64": 0.8082578808157032, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.3369255514705882, + "pass@2": 0.440790368805002, + "pass@4": 0.5347838508237939, + "pass@8": 0.6161963128406492, + "pass@16": 0.6852001345712164, + "pass@32": 0.7416791147734874, + "pass@64": 0.7868330160139833, + "pass@128": 0.8235294117647058 + }, + "19": { + "pass@1": 0.21549479166666666, + "pass@2": 0.29482081341394845, + "pass@4": 0.3771499184811719, + "pass@8": 0.45314763547808923, + "pass@16": 0.5185264418909887, + "pass@32": 0.5747617192946216, + "pass@64": 0.6211208246521323, + "pass@128": 0.6517857142857143 + }, + "20": { + "pass@1": 0.18099738023952097, + "pass@2": 0.25491352432929404, + "pass@4": 0.33130779498371077, + "pass@8": 0.4034410216749828, + "pass@16": 0.4688099342745726, + "pass@32": 0.5279449481851225, + "pass@64": 0.5843637752328384, + "pass@128": 0.6377245508982036 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 236538, + "answer_accuracy": 0.5647778499388753, + "avg_response_len": 377.49371084199265, + "resp_tokens_sum": 158100406, + "pass_at_k": { + "pass@1": 0.5647778499388753, + "pass@2": 0.6368192583095919, + "pass@4": 0.697527831503492, + "pass@8": 0.7462034965586087, + "pass@16": 0.783661892458126, + "pass@32": 0.8127123031694083, + "pass@64": 0.8355373694418495, + "pass@128": 0.8542176039119804 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9620535714285714, + "pass@2": 0.9741579177602799, + "pass@4": 0.9804710363585502, + "pass@8": 0.984405510081136, + "pass@16": 0.9870437043502408, + "pass@32": 0.9885943947975548, + "pass@64": 0.9896883459298144, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.907157874617737, + "pass@2": 0.9334794661561804, + "pass@4": 0.9506061639084101, + "pass@8": 0.9616284980049788, + "pass@16": 0.9681187712875272, + "pass@32": 0.9718688464597104, + "pass@64": 0.9743606390761322, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7944396219135802, + "pass@2": 0.8480640766258388, + "pass@4": 0.883981797992843, + "pass@8": 0.9083938480790773, + "pass@16": 0.9249280339745628, + "pass@32": 0.9374717148724703, + "pass@64": 0.9475575890272085, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.771281528189911, + "pass@2": 0.8348344149396016, + "pass@4": 0.87485633286937, + "pass@8": 0.9029845078915546, + "pass@16": 0.9249497229959522, + "pass@32": 0.9420130044854153, + "pass@64": 0.9537250990028946, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.6810378086419753, + "pass@2": 0.759533801764363, + "pass@4": 0.8115666965240457, + "pass@8": 0.8475286718805226, + "pass@16": 0.8743301152204945, + "pass@32": 0.8953343449790683, + "pass@64": 0.9097404523733096, + "pass@128": 0.9166666666666666 + }, + "16": { + "pass@1": 0.4934089233038348, + "pass@2": 0.6187224903026508, + "pass@4": 0.720293994500687, + "pass@8": 0.7920491057615308, + "pass@16": 0.836986988210621, + "pass@32": 0.866034039911091, + "pass@64": 0.889947041964446, + "pass@128": 0.9144542772861357 + }, + "17": { + "pass@1": 0.3518684111445783, + "pass@2": 0.45531103251589033, + "pass@4": 0.5494780251489645, + "pass@8": 0.6294890131094445, + "pass@16": 0.6943459390943068, + "pass@32": 0.7455723174815397, + "pass@64": 0.7875108943405223, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.3236139112903226, + "pass@2": 0.41808364554229105, + "pass@4": 0.5064354890114543, + "pass@8": 0.5804429288858804, + "pass@16": 0.6393856336344137, + "pass@32": 0.6878125541269339, + "pass@64": 0.7277094054798247, + "pass@128": 0.7612903225806451 + }, + "19": { + "pass@1": 0.20324301359516617, + "pass@2": 0.28858765789661495, + "pass@4": 0.3778401204381177, + "pass@8": 0.4559778835038485, + "pass@16": 0.5186360320155117, + "pass@32": 0.5718077443113075, + "pass@64": 0.6175381100536046, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.17600412912912913, + "pass@2": 0.2502268508666147, + "pass@4": 0.3296207996522957, + "pass@8": 0.40676253450542404, + "pass@16": 0.4739966854242499, + "pass@32": 0.5258046811806326, + "pass@64": 0.5621618523024845, + "pass@128": 0.5915915915915916 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7743569078947369, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7743569078947369, + "pass@2": 0.8125631604848585, + "pass@4": 0.8437409665946201, + "pass@8": 0.868623288598324, + "pass@16": 0.8883868086355705, + "pass@32": 0.9042106156511917, + "pass@64": 0.91704470785007, + "pass@128": 0.9276315789473685 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9566953125, + "pass@2": 0.9671897145669298, + "pass@4": 0.9763015637420321, + "pass@8": 0.9841177284807338, + "pass@16": 0.9898433434950407, + "pass@32": 0.9935109453775409, + "pass@64": 0.9956717768827965, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9993671875, + "pass@2": 0.9999009596456694, + "pass@4": 0.9999937616235471, + "pass@8": 0.9999999729956526, + "pass@16": 0.9999999999998621, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99846875, + "pass@2": 0.9992100147637796, + "pass@4": 0.9997477704349457, + "pass@8": 0.9999643299327841, + "pass@16": 0.9999991554454004, + "pass@32": 0.9999999998305314, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936171875, + "pass@2": 0.9961348425196852, + "pass@4": 0.9976905641169854, + "pass@8": 0.9985067082658915, + "pass@16": 0.9989825851297969, + "pass@32": 0.9993892496351895, + "pass@64": 0.9997513007685047, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9937734375, + "pass@2": 0.9954635826771654, + "pass@4": 0.9965071167041619, + "pass@8": 0.9974872765717031, + "pass@16": 0.9985219008892885, + "pass@32": 0.9994581520603246, + "pass@64": 0.9999507579626357, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.957140625, + "pass@2": 0.9682801427165355, + "pass@4": 0.9749531122047241, + "pass@8": 0.9791891767647783, + "pass@16": 0.98211903213778, + "pass@32": 0.9846358344909678, + "pass@64": 0.9867111407997167, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9026015625, + "pass@2": 0.924557701771654, + "pass@4": 0.9376451166104238, + "pass@8": 0.9468465265856568, + "pass@16": 0.954191808841156, + "pass@32": 0.9603990401824475, + "pass@64": 0.9657758332302137, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.814109375, + "pass@2": 0.8641605561023634, + "pass@4": 0.895520004874391, + "pass@8": 0.9164012838141359, + "pass@16": 0.9319405625887556, + "pass@32": 0.9444791104314104, + "pass@64": 0.9538452723690258, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7910234375, + "pass@2": 0.8541248769685043, + "pass@4": 0.8911382658417697, + "pass@8": 0.9145212717761322, + "pass@16": 0.9314696532384339, + "pass@32": 0.9443763740796397, + "pass@64": 0.9539442507806651, + "pass@128": 0.961 + }, + "15": { + "pass@1": 0.6977265625, + "pass@2": 0.782586245078741, + "pass@4": 0.8406202425946752, + "pass@8": 0.8807529357498628, + "pass@16": 0.9092865874591104, + "pass@32": 0.9296244730051035, + "pass@64": 0.9429202877342062, + "pass@128": 0.951 + }, + "16": { + "pass@1": 0.508703125, + "pass@2": 0.6256612942913391, + "pass@4": 0.7164985766779153, + "pass@8": 0.7826049747381607, + "pass@16": 0.8290852993476789, + "pass@32": 0.8612613381637253, + "pass@64": 0.8844828159208924, + "pass@128": 0.903 + }, + "17": { + "pass@1": 0.3892578125, + "pass@2": 0.4965643454724407, + "pass@4": 0.5878991002062244, + "pass@8": 0.6593557882776578, + "pass@16": 0.7147717915717123, + "pass@32": 0.7588696172178708, + "pass@64": 0.7963482777124878, + "pass@128": 0.831 + }, + "18": { + "pass@1": 0.320015625, + "pass@2": 0.4193448572834649, + "pass@4": 0.5105761571053615, + "pass@8": 0.5890740631550005, + "pass@16": 0.6547608784288393, + "pass@32": 0.7090559625862405, + "pass@64": 0.7541917451827876, + "pass@128": 0.793 + }, + "19": { + "pass@1": 0.208, + "pass@2": 0.28776230314960616, + "pass@4": 0.3691554985939253, + "pass@8": 0.44260925321184885, + "pass@16": 0.5048892048968053, + "pass@32": 0.5590981633198217, + "pass@64": 0.6067321047916664, + "pass@128": 0.645 + }, + "20": { + "pass@1": 0.1824375, + "pass@2": 0.2577630413385826, + "pass@4": 0.33683151677915263, + "pass@8": 0.41241119304797336, + "pass@16": 0.47948756060617204, + "pass@32": 0.5358434369915598, + "pass@64": 0.5835238850156994, + "pass@128": 0.626 + } + }, + "per_op_accuracy": { + "10": 0.9566953125, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.9993671875, + "6": 0.99846875, + "7": 0.9999140625, + "8": 0.9936171875, + "9": 0.9937734375, + "11": 0.957140625, + "12": 0.9026015625, + "13": 0.814109375, + "14": 0.7910234375, + "15": 0.6977265625, + "16": 0.508703125, + "17": 0.3892578125, + "18": 0.320015625, + "19": 0.208, + "20": 0.1824375 + }, + "per_op_avg_response_len": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375, + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375, + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 628467, + "answer_accuracy": 0.7761458168668985, + "avg_response_len": 268.07536604884604, + "resp_tokens_sum": 217068130, + "pass_at_k": { + "pass@1": 0.7761458168668985, + "pass@2": 0.8139650783480664, + "pass@4": 0.844180951782302, + "pass@8": 0.868240403688873, + "pass@16": 0.8877728581338872, + "pass@32": 0.9036649765222275, + "pass@64": 0.9170053125522537, + "pass@128": 0.9287069238065128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9618132374631269, + "pass@2": 0.9725411121176223, + "pass@4": 0.9802102696233768, + "pass@8": 0.9849936946412662, + "pass@16": 0.9884440697041347, + "pass@32": 0.991615765648179, + "pass@64": 0.9937203955529144, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.999994232898622, + "pass@4": 0.9999999956060179, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996279761904762, + "pass@2": 0.9999808617672791, + "pass@4": 0.9999999687539058, + "pass@8": 0.9999999999999978, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985965568862275, + "pass@2": 0.9993480067424206, + "pass@4": 0.9998631435105043, + "pass@8": 0.9999946418638593, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999934480268369, + "pass@4": 0.999999995840017, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9912359775641025, + "pass@2": 0.9958788392136078, + "pass@4": 0.9986500876092409, + "pass@8": 0.9997715179680895, + "pass@16": 0.9999920381521239, + "pass@32": 0.9999999947732479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972881085726961, + "pass@4": 0.9977522703148021, + "pass@8": 0.9982272475790946, + "pass@16": 0.998920396325759, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9514651203966006, + "pass@2": 0.964051451562535, + "pass@4": 0.9726533114840814, + "pass@8": 0.9784351301207374, + "pass@16": 0.9821156095662544, + "pass@32": 0.9847244967783608, + "pass@64": 0.9857797640615148, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8961864406779662, + "pass@2": 0.9169464416789005, + "pass@4": 0.930045741104395, + "pass@8": 0.9405154502869694, + "pass@16": 0.9496226622064528, + "pass@32": 0.9572528675630418, + "pass@64": 0.9642242554401338, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8420111677115988, + "pass@2": 0.889376943820502, + "pass@4": 0.9181533575936236, + "pass@8": 0.9363334327363076, + "pass@16": 0.949785723044496, + "pass@32": 0.9610582398326716, + "pass@64": 0.9701180751617702, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8046388239875389, + "pass@2": 0.8602070915201021, + "pass@4": 0.8914461434143157, + "pass@8": 0.9119244313646143, + "pass@16": 0.9272654211635459, + "pass@32": 0.9383072121466731, + "pass@64": 0.9462294718077346, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.7136244580924855, + "pass@2": 0.8005655892995307, + "pass@4": 0.8601507764022269, + "pass@8": 0.9013584538970695, + "pass@16": 0.9310673045046043, + "pass@32": 0.9517436419640584, + "pass@64": 0.9651808484660429, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.523109879032258, + "pass@2": 0.6348286290322582, + "pass@4": 0.7166433194842577, + "pass@8": 0.7779303950852562, + "pass@16": 0.8253214388050136, + "pass@32": 0.8612953308847185, + "pass@64": 0.8875623036766475, + "pass@128": 0.9032258064516129 + }, + "17": { + "pass@1": 0.40713059971098264, + "pass@2": 0.5174622940466982, + "pass@4": 0.6078873918758712, + "pass@8": 0.6741004742749396, + "pass@16": 0.7231938373226637, + "pass@32": 0.7607190313851854, + "pass@64": 0.7937445756323053, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.3004017857142857, + "pass@2": 0.3996291479190101, + "pass@4": 0.4907275606620602, + "pass@8": 0.5703714538130185, + "pass@16": 0.6388093892798813, + "pass@32": 0.6961804908111601, + "pass@64": 0.7459388686836753, + "pass@128": 0.7914285714285715 + }, + "19": { + "pass@1": 0.2051661036036036, + "pass@2": 0.27981980504126164, + "pass@4": 0.35245659495265824, + "pass@8": 0.4186875923466568, + "pass@16": 0.4774648463192164, + "pass@32": 0.530660247056415, + "pass@64": 0.5814726524949139, + "pass@128": 0.6306306306306306 + }, + "20": { + "pass@1": 0.19031531531531531, + "pass@2": 0.26815730592797515, + "pass@4": 0.3495825434545906, + "pass@8": 0.42705696041508395, + "pass@16": 0.4956881270636332, + "pass@32": 0.5538044007945283, + "pass@64": 0.6040435053189296, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 638217, + "answer_accuracy": 0.7772518024162121, + "avg_response_len": 277.55285585541696, + "resp_tokens_sum": 227904201, + "pass_at_k": { + "pass@1": 0.7772518024162121, + "pass@2": 0.815610040597508, + "pass@4": 0.8465593161240585, + "pass@8": 0.8711925292394912, + "pass@16": 0.8910398237778925, + "pass@32": 0.9072447831633289, + "pass@64": 0.9203445261243395, + "pass@128": 0.9304754481683554 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9461752300613497, + "pass@2": 0.9579337954688179, + "pass@4": 0.9694229502945566, + "pass@8": 0.9810037051895499, + "pass@16": 0.9905673067988211, + "pass@32": 0.9965809863505134, + "pass@64": 0.9993884580111478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9990942028985508, + "pass@2": 0.9997621391076116, + "pass@4": 0.9999820552322264, + "pass@8": 0.9999999217268679, + "pass@16": 0.9999999999996003, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980524713467048, + "pass@2": 0.9987414829772353, + "pass@4": 0.9994561608344803, + "pass@8": 0.9999035515654319, + "pass@16": 0.9999975847114193, + "pass@32": 0.9999999995144166, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965228873239437, + "pass@2": 0.9971952284573583, + "pass@4": 0.9973554949117276, + "pass@8": 0.997525506974028, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928190494011976, + "pass@2": 0.9945048387477016, + "pass@4": 0.9955588426633797, + "pass@8": 0.996690741627332, + "pass@16": 0.9980681831422826, + "pass@32": 0.9992670805169979, + "pass@64": 0.9999136334847853, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9585137424698795, + "pass@2": 0.9671995066881702, + "pass@4": 0.9721630024033138, + "pass@8": 0.9750416872186753, + "pass@16": 0.9774501659354414, + "pass@32": 0.9807857011053507, + "pass@64": 0.9848766720425005, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9050499608150471, + "pass@2": 0.923858545405179, + "pass@4": 0.9327921901611824, + "pass@8": 0.9387195559136081, + "pass@16": 0.9449860445424791, + "pass@32": 0.952132953833246, + "pass@64": 0.9586975480454887, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8070290616246498, + "pass@2": 0.8562368352852956, + "pass@4": 0.8857674544883569, + "pass@8": 0.9058580167888862, + "pass@16": 0.9223591986829233, + "pass@32": 0.9360243594008619, + "pass@64": 0.9450110575563182, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7976973684210527, + "pass@2": 0.867424569461712, + "pass@4": 0.9068931860491114, + "pass@8": 0.9283267548205743, + "pass@16": 0.9418403403371376, + "pass@32": 0.9524016855817911, + "pass@64": 0.9614012630596699, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6974431818181818, + "pass@2": 0.7863684830589354, + "pass@4": 0.8486680735930725, + "pass@8": 0.891768548521778, + "pass@16": 0.9207707356638722, + "pass@32": 0.9400995306433965, + "pass@64": 0.9521569927151597, + "pass@128": 0.9606060606060606 + }, + "16": { + "pass@1": 0.5107505341880342, + "pass@2": 0.6242663677456984, + "pass@4": 0.7127050812024565, + "pass@8": 0.7776122661212889, + "pass@16": 0.8247779610106144, + "pass@32": 0.8566217836455938, + "pass@64": 0.8764856255133433, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4086034549689441, + "pass@2": 0.5166432575683475, + "pass@4": 0.6060352741233433, + "pass@8": 0.6743064342428341, + "pass@16": 0.7267822114247295, + "pass@32": 0.7705926178718194, + "pass@64": 0.8082578808157032, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.3369255514705882, + "pass@2": 0.440790368805002, + "pass@4": 0.5347838508237939, + "pass@8": 0.6161963128406492, + "pass@16": 0.6852001345712164, + "pass@32": 0.7416791147734874, + "pass@64": 0.7868330160139833, + "pass@128": 0.8235294117647058 + }, + "19": { + "pass@1": 0.21549479166666666, + "pass@2": 0.29482081341394845, + "pass@4": 0.3771499184811719, + "pass@8": 0.45314763547808923, + "pass@16": 0.5185264418909887, + "pass@32": 0.5747617192946216, + "pass@64": 0.6211208246521323, + "pass@128": 0.6517857142857143 + }, + "20": { + "pass@1": 0.18099738023952097, + "pass@2": 0.25491352432929404, + "pass@4": 0.33130779498371077, + "pass@8": 0.4034410216749828, + "pass@16": 0.4688099342745726, + "pass@32": 0.5279449481851225, + "pass@64": 0.5843637752328384, + "pass@128": 0.6377245508982036 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 616552, + "answer_accuracy": 0.7695818022048251, + "avg_response_len": 298.12597234981627, + "resp_tokens_sum": 238844219, + "pass_at_k": { + "pass@1": 0.7695818022048251, + "pass@2": 0.8080234147866366, + "pass@4": 0.8404076771668446, + "pass@8": 0.8663769954562627, + "pass@16": 0.8862881920411901, + "pass@32": 0.901652304185975, + "pass@64": 0.9137024615526762, + "pass@128": 0.9236299728391116 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9617537313432836, + "pass@2": 0.970781672346927, + "pass@4": 0.9790400016229313, + "pass@8": 0.98626166598672, + "pass@16": 0.9905548114893244, + "pass@32": 0.9924411918583297, + "pass@64": 0.9940296282947009, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994025735294118, + "pass@2": 0.9999677946966188, + "pass@4": 0.9999998894439663, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987923895899053, + "pass@2": 0.999580450955066, + "pass@4": 0.999947256691494, + "pass@8": 0.9999993061637832, + "pass@16": 0.9999999999491976, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927505630630631, + "pass@2": 0.9952442600080397, + "pass@4": 0.9971487570247412, + "pass@8": 0.9983676867389416, + "pass@16": 0.9992487477653507, + "pass@32": 0.9998506753392894, + "pass@64": 0.9999979947884918, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925391881028939, + "pass@2": 0.9944105723219485, + "pass@4": 0.9961042035139499, + "pass@8": 0.997498057805773, + "pass@16": 0.9985542991129313, + "pass@32": 0.9994502950777389, + "pass@64": 0.9999555508074901, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9620535714285714, + "pass@2": 0.9741579177602799, + "pass@4": 0.9804710363585502, + "pass@8": 0.984405510081136, + "pass@16": 0.9870437043502408, + "pass@32": 0.9885943947975548, + "pass@64": 0.9896883459298144, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.907157874617737, + "pass@2": 0.9334794661561804, + "pass@4": 0.9506061639084101, + "pass@8": 0.9616284980049788, + "pass@16": 0.9681187712875272, + "pass@32": 0.9718688464597104, + "pass@64": 0.9743606390761322, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7944396219135802, + "pass@2": 0.8480640766258388, + "pass@4": 0.883981797992843, + "pass@8": 0.9083938480790773, + "pass@16": 0.9249280339745628, + "pass@32": 0.9374717148724703, + "pass@64": 0.9475575890272085, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.771281528189911, + "pass@2": 0.8348344149396016, + "pass@4": 0.87485633286937, + "pass@8": 0.9029845078915546, + "pass@16": 0.9249497229959522, + "pass@32": 0.9420130044854153, + "pass@64": 0.9537250990028946, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.6810378086419753, + "pass@2": 0.759533801764363, + "pass@4": 0.8115666965240457, + "pass@8": 0.8475286718805226, + "pass@16": 0.8743301152204945, + "pass@32": 0.8953343449790683, + "pass@64": 0.9097404523733096, + "pass@128": 0.9166666666666666 + }, + "16": { + "pass@1": 0.4934089233038348, + "pass@2": 0.6187224903026508, + "pass@4": 0.720293994500687, + "pass@8": 0.7920491057615308, + "pass@16": 0.836986988210621, + "pass@32": 0.866034039911091, + "pass@64": 0.889947041964446, + "pass@128": 0.9144542772861357 + }, + "17": { + "pass@1": 0.3518684111445783, + "pass@2": 0.45531103251589033, + "pass@4": 0.5494780251489645, + "pass@8": 0.6294890131094445, + "pass@16": 0.6943459390943068, + "pass@32": 0.7455723174815397, + "pass@64": 0.7875108943405223, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.3236139112903226, + "pass@2": 0.41808364554229105, + "pass@4": 0.5064354890114543, + "pass@8": 0.5804429288858804, + "pass@16": 0.6393856336344137, + "pass@32": 0.6878125541269339, + "pass@64": 0.7277094054798247, + "pass@128": 0.7612903225806451 + }, + "19": { + "pass@1": 0.20324301359516617, + "pass@2": 0.28858765789661495, + "pass@4": 0.3778401204381177, + "pass@8": 0.4559778835038485, + "pass@16": 0.5186360320155117, + "pass@32": 0.5718077443113075, + "pass@64": 0.6175381100536046, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.17600412912912913, + "pass@2": 0.2502268508666147, + "pass@4": 0.3296207996522957, + "pass@8": 0.40676253450542404, + "pass@16": 0.4739966854242499, + "pass@32": 0.5258046811806326, + "pass@64": 0.5621618523024845, + "pass@128": 0.5915915915915916 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl b/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4f0830d7ab45aa040e9c6bc874584db9e402770a --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:526ade9e76104f63e7675b250bcd8567568aec946a997d8b41a4f212d515fa92 +size 2647783473 diff --git a/results/composition/SDAR-100M-block8-epoch4-op9-12/summary.csv b/results/composition/SDAR-100M-block8-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d85ae124934bcf76697f95e77d65568554189806 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op9-12,0.9935295138888889,0.5771015625,0.7743569078947369,0.0,0.0,0.0,199.78453819444445,354.4255953125,1152000,1280000,2432000 diff --git a/results/composition/SDAR-100M-block8-epoch4-op9-12/summary.json b/results/composition/SDAR-100M-block8-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b84ff3d99850fc8ac0ff7bc73e788019a37cc795 --- /dev/null +++ b/results/composition/SDAR-100M-block8-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op9-12", + "id_acc": 0.9935295138888889, + "ood_acc": 0.5771015625, + "total_acc": 0.7743569078947369, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.78453819444445, + "ood_avg_resp_len": 354.4255953125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 628467, + "answer_accuracy": 0.7761458168668985, + "avg_response_len": 268.07536604884604, + "resp_tokens_sum": 217068130, + "pass_at_k": { + "pass@1": 0.7761458168668985, + "pass@2": 0.8139650783480664, + "pass@4": 0.844180951782302, + "pass@8": 0.868240403688873, + "pass@16": 0.8877728581338872, + "pass@32": 0.9036649765222275, + "pass@64": 0.9170053125522537, + "pass@128": 0.9287069238065128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9618132374631269, + "pass@2": 0.9725411121176223, + "pass@4": 0.9802102696233768, + "pass@8": 0.9849936946412662, + "pass@16": 0.9884440697041347, + "pass@32": 0.991615765648179, + "pass@64": 0.9937203955529144, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.999994232898622, + "pass@4": 0.9999999956060179, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996279761904762, + "pass@2": 0.9999808617672791, + "pass@4": 0.9999999687539058, + "pass@8": 0.9999999999999978, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985965568862275, + "pass@2": 0.9993480067424206, + "pass@4": 0.9998631435105043, + "pass@8": 0.9999946418638593, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999934480268369, + "pass@4": 0.999999995840017, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9912359775641025, + "pass@2": 0.9958788392136078, + "pass@4": 0.9986500876092409, + "pass@8": 0.9997715179680895, + "pass@16": 0.9999920381521239, + "pass@32": 0.9999999947732479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972881085726961, + "pass@4": 0.9977522703148021, + "pass@8": 0.9982272475790946, + "pass@16": 0.998920396325759, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9514651203966006, + "pass@2": 0.964051451562535, + "pass@4": 0.9726533114840814, + "pass@8": 0.9784351301207374, + "pass@16": 0.9821156095662544, + "pass@32": 0.9847244967783608, + "pass@64": 0.9857797640615148, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8961864406779662, + "pass@2": 0.9169464416789005, + "pass@4": 0.930045741104395, + "pass@8": 0.9405154502869694, + "pass@16": 0.9496226622064528, + "pass@32": 0.9572528675630418, + "pass@64": 0.9642242554401338, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8420111677115988, + "pass@2": 0.889376943820502, + "pass@4": 0.9181533575936236, + "pass@8": 0.9363334327363076, + "pass@16": 0.949785723044496, + "pass@32": 0.9610582398326716, + "pass@64": 0.9701180751617702, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8046388239875389, + "pass@2": 0.8602070915201021, + "pass@4": 0.8914461434143157, + "pass@8": 0.9119244313646143, + "pass@16": 0.9272654211635459, + "pass@32": 0.9383072121466731, + "pass@64": 0.9462294718077346, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.7136244580924855, + "pass@2": 0.8005655892995307, + "pass@4": 0.8601507764022269, + "pass@8": 0.9013584538970695, + "pass@16": 0.9310673045046043, + "pass@32": 0.9517436419640584, + "pass@64": 0.9651808484660429, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.523109879032258, + "pass@2": 0.6348286290322582, + "pass@4": 0.7166433194842577, + "pass@8": 0.7779303950852562, + "pass@16": 0.8253214388050136, + "pass@32": 0.8612953308847185, + "pass@64": 0.8875623036766475, + "pass@128": 0.9032258064516129 + }, + "17": { + "pass@1": 0.40713059971098264, + "pass@2": 0.5174622940466982, + "pass@4": 0.6078873918758712, + "pass@8": 0.6741004742749396, + "pass@16": 0.7231938373226637, + "pass@32": 0.7607190313851854, + "pass@64": 0.7937445756323053, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.3004017857142857, + "pass@2": 0.3996291479190101, + "pass@4": 0.4907275606620602, + "pass@8": 0.5703714538130185, + "pass@16": 0.6388093892798813, + "pass@32": 0.6961804908111601, + "pass@64": 0.7459388686836753, + "pass@128": 0.7914285714285715 + }, + "19": { + "pass@1": 0.2051661036036036, + "pass@2": 0.27981980504126164, + "pass@4": 0.35245659495265824, + "pass@8": 0.4186875923466568, + "pass@16": 0.4774648463192164, + "pass@32": 0.530660247056415, + "pass@64": 0.5814726524949139, + "pass@128": 0.6306306306306306 + }, + "20": { + "pass@1": 0.19031531531531531, + "pass@2": 0.26815730592797515, + "pass@4": 0.3495825434545906, + "pass@8": 0.42705696041508395, + "pass@16": 0.4956881270636332, + "pass@32": 0.5538044007945283, + "pass@64": 0.6040435053189296, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 638217, + "answer_accuracy": 0.7772518024162121, + "avg_response_len": 277.55285585541696, + "resp_tokens_sum": 227904201, + "pass_at_k": { + "pass@1": 0.7772518024162121, + "pass@2": 0.815610040597508, + "pass@4": 0.8465593161240585, + "pass@8": 0.8711925292394912, + "pass@16": 0.8910398237778925, + "pass@32": 0.9072447831633289, + "pass@64": 0.9203445261243395, + "pass@128": 0.9304754481683554 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9461752300613497, + "pass@2": 0.9579337954688179, + "pass@4": 0.9694229502945566, + "pass@8": 0.9810037051895499, + "pass@16": 0.9905673067988211, + "pass@32": 0.9965809863505134, + "pass@64": 0.9993884580111478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9990942028985508, + "pass@2": 0.9997621391076116, + "pass@4": 0.9999820552322264, + "pass@8": 0.9999999217268679, + "pass@16": 0.9999999999996003, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980524713467048, + "pass@2": 0.9987414829772353, + "pass@4": 0.9994561608344803, + "pass@8": 0.9999035515654319, + "pass@16": 0.9999975847114193, + "pass@32": 0.9999999995144166, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965228873239437, + "pass@2": 0.9971952284573583, + "pass@4": 0.9973554949117276, + "pass@8": 0.997525506974028, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928190494011976, + "pass@2": 0.9945048387477016, + "pass@4": 0.9955588426633797, + "pass@8": 0.996690741627332, + "pass@16": 0.9980681831422826, + "pass@32": 0.9992670805169979, + "pass@64": 0.9999136334847853, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9585137424698795, + "pass@2": 0.9671995066881702, + "pass@4": 0.9721630024033138, + "pass@8": 0.9750416872186753, + "pass@16": 0.9774501659354414, + "pass@32": 0.9807857011053507, + "pass@64": 0.9848766720425005, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9050499608150471, + "pass@2": 0.923858545405179, + "pass@4": 0.9327921901611824, + "pass@8": 0.9387195559136081, + "pass@16": 0.9449860445424791, + "pass@32": 0.952132953833246, + "pass@64": 0.9586975480454887, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8070290616246498, + "pass@2": 0.8562368352852956, + "pass@4": 0.8857674544883569, + "pass@8": 0.9058580167888862, + "pass@16": 0.9223591986829233, + "pass@32": 0.9360243594008619, + "pass@64": 0.9450110575563182, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7976973684210527, + "pass@2": 0.867424569461712, + "pass@4": 0.9068931860491114, + "pass@8": 0.9283267548205743, + "pass@16": 0.9418403403371376, + "pass@32": 0.9524016855817911, + "pass@64": 0.9614012630596699, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6974431818181818, + "pass@2": 0.7863684830589354, + "pass@4": 0.8486680735930725, + "pass@8": 0.891768548521778, + "pass@16": 0.9207707356638722, + "pass@32": 0.9400995306433965, + "pass@64": 0.9521569927151597, + "pass@128": 0.9606060606060606 + }, + "16": { + "pass@1": 0.5107505341880342, + "pass@2": 0.6242663677456984, + "pass@4": 0.7127050812024565, + "pass@8": 0.7776122661212889, + "pass@16": 0.8247779610106144, + "pass@32": 0.8566217836455938, + "pass@64": 0.8764856255133433, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4086034549689441, + "pass@2": 0.5166432575683475, + "pass@4": 0.6060352741233433, + "pass@8": 0.6743064342428341, + "pass@16": 0.7267822114247295, + "pass@32": 0.7705926178718194, + "pass@64": 0.8082578808157032, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.3369255514705882, + "pass@2": 0.440790368805002, + "pass@4": 0.5347838508237939, + "pass@8": 0.6161963128406492, + "pass@16": 0.6852001345712164, + "pass@32": 0.7416791147734874, + "pass@64": 0.7868330160139833, + "pass@128": 0.8235294117647058 + }, + "19": { + "pass@1": 0.21549479166666666, + "pass@2": 0.29482081341394845, + "pass@4": 0.3771499184811719, + "pass@8": 0.45314763547808923, + "pass@16": 0.5185264418909887, + "pass@32": 0.5747617192946216, + "pass@64": 0.6211208246521323, + "pass@128": 0.6517857142857143 + }, + "20": { + "pass@1": 0.18099738023952097, + "pass@2": 0.25491352432929404, + "pass@4": 0.33130779498371077, + "pass@8": 0.4034410216749828, + "pass@16": 0.4688099342745726, + "pass@32": 0.5279449481851225, + "pass@64": 0.5843637752328384, + "pass@128": 0.6377245508982036 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 616552, + "answer_accuracy": 0.7695818022048251, + "avg_response_len": 298.12597234981627, + "resp_tokens_sum": 238844219, + "pass_at_k": { + "pass@1": 0.7695818022048251, + "pass@2": 0.8080234147866366, + "pass@4": 0.8404076771668446, + "pass@8": 0.8663769954562627, + "pass@16": 0.8862881920411901, + "pass@32": 0.901652304185975, + "pass@64": 0.9137024615526762, + "pass@128": 0.9236299728391116 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9617537313432836, + "pass@2": 0.970781672346927, + "pass@4": 0.9790400016229313, + "pass@8": 0.98626166598672, + "pass@16": 0.9905548114893244, + "pass@32": 0.9924411918583297, + "pass@64": 0.9940296282947009, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994025735294118, + "pass@2": 0.9999677946966188, + "pass@4": 0.9999998894439663, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987923895899053, + "pass@2": 0.999580450955066, + "pass@4": 0.999947256691494, + "pass@8": 0.9999993061637832, + "pass@16": 0.9999999999491976, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927505630630631, + "pass@2": 0.9952442600080397, + "pass@4": 0.9971487570247412, + "pass@8": 0.9983676867389416, + "pass@16": 0.9992487477653507, + "pass@32": 0.9998506753392894, + "pass@64": 0.9999979947884918, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925391881028939, + "pass@2": 0.9944105723219485, + "pass@4": 0.9961042035139499, + "pass@8": 0.997498057805773, + "pass@16": 0.9985542991129313, + "pass@32": 0.9994502950777389, + "pass@64": 0.9999555508074901, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9620535714285714, + "pass@2": 0.9741579177602799, + "pass@4": 0.9804710363585502, + "pass@8": 0.984405510081136, + "pass@16": 0.9870437043502408, + "pass@32": 0.9885943947975548, + "pass@64": 0.9896883459298144, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.907157874617737, + "pass@2": 0.9334794661561804, + "pass@4": 0.9506061639084101, + "pass@8": 0.9616284980049788, + "pass@16": 0.9681187712875272, + "pass@32": 0.9718688464597104, + "pass@64": 0.9743606390761322, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7944396219135802, + "pass@2": 0.8480640766258388, + "pass@4": 0.883981797992843, + "pass@8": 0.9083938480790773, + "pass@16": 0.9249280339745628, + "pass@32": 0.9374717148724703, + "pass@64": 0.9475575890272085, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.771281528189911, + "pass@2": 0.8348344149396016, + "pass@4": 0.87485633286937, + "pass@8": 0.9029845078915546, + "pass@16": 0.9249497229959522, + "pass@32": 0.9420130044854153, + "pass@64": 0.9537250990028946, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.6810378086419753, + "pass@2": 0.759533801764363, + "pass@4": 0.8115666965240457, + "pass@8": 0.8475286718805226, + "pass@16": 0.8743301152204945, + "pass@32": 0.8953343449790683, + "pass@64": 0.9097404523733096, + "pass@128": 0.9166666666666666 + }, + "16": { + "pass@1": 0.4934089233038348, + "pass@2": 0.6187224903026508, + "pass@4": 0.720293994500687, + "pass@8": 0.7920491057615308, + "pass@16": 0.836986988210621, + "pass@32": 0.866034039911091, + "pass@64": 0.889947041964446, + "pass@128": 0.9144542772861357 + }, + "17": { + "pass@1": 0.3518684111445783, + "pass@2": 0.45531103251589033, + "pass@4": 0.5494780251489645, + "pass@8": 0.6294890131094445, + "pass@16": 0.6943459390943068, + "pass@32": 0.7455723174815397, + "pass@64": 0.7875108943405223, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.3236139112903226, + "pass@2": 0.41808364554229105, + "pass@4": 0.5064354890114543, + "pass@8": 0.5804429288858804, + "pass@16": 0.6393856336344137, + "pass@32": 0.6878125541269339, + "pass@64": 0.7277094054798247, + "pass@128": 0.7612903225806451 + }, + "19": { + "pass@1": 0.20324301359516617, + "pass@2": 0.28858765789661495, + "pass@4": 0.3778401204381177, + "pass@8": 0.4559778835038485, + "pass@16": 0.5186360320155117, + "pass@32": 0.5718077443113075, + "pass@64": 0.6175381100536046, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.17600412912912913, + "pass@2": 0.2502268508666147, + "pass@4": 0.3296207996522957, + "pass@8": 0.40676253450542404, + "pass@16": 0.4739966854242499, + "pass@32": 0.5258046811806326, + "pass@64": 0.5621618523024845, + "pass@128": 0.5915915915915916 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..031fe745ac0b0520a153eb4eb8899bd48026ed1e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b07036a063d8787116d37dd0ad45218123a93758079ed5fc24c8b0fcd57422cb +size 1793632655 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..54b3160ea5b742632d4941713ad743b9e18178ba --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op11-14", + "id": { + "answer_accuracy": 0.967625, + "avg_response_len": 200.41735677083332, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.967625, + "pass@2": 0.9789966644794311, + "pass@4": 0.9855308038266036, + "pass@8": 0.9898741675084467, + "pass@16": 0.9928107492930721, + "pass@32": 0.9948438671477686, + "pass@64": 0.9964485740453495, + "pass@128": 0.9978888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9061171875, + "pass@2": 0.9296520669291347, + "pass@4": 0.946490177071616, + "pass@8": 0.9600063218340427, + "pass@16": 0.9701224203043363, + "pass@32": 0.9773745044873484, + "pass@64": 0.9830228101934626, + "pass@128": 0.988 + }, + "2": { + "pass@1": 0.9942578125, + "pass@2": 0.9963058562992128, + "pass@4": 0.9973353727034121, + "pass@8": 0.9980348689327625, + "pass@16": 0.9986730600533266, + "pass@32": 0.9992581488007652, + "pass@64": 0.9997381327355772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.992875, + "pass@2": 0.99627374507874, + "pass@4": 0.9983502170041246, + "pass@8": 0.9993787301566442, + "pass@16": 0.9998215164710738, + "pass@32": 0.99998079290106, + "pass@64": 0.9999999379483816, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9973515625, + "pass@2": 0.9992237942913387, + "pass@4": 0.999845853112111, + "pass@8": 0.9999885322027398, + "pass@16": 0.9999999276229732, + "pass@32": 0.9999999999993945, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9612109375, + "pass@2": 0.9706935285433075, + "pass@4": 0.9771380594300711, + "pass@8": 0.9823009579962168, + "pass@16": 0.9866195147329673, + "pass@32": 0.9902375379909053, + "pass@64": 0.9934479880732294, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9711171875, + "pass@2": 0.9824745324803148, + "pass@4": 0.9884285929883763, + "pass@8": 0.9915199624038011, + "pass@16": 0.9934720350654136, + "pass@32": 0.9949225786814185, + "pass@64": 0.9959341500065584, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.974828125, + "pass@2": 0.9842936761811031, + "pass@4": 0.989352229846269, + "pass@8": 0.9928827326004378, + "pass@16": 0.9950953202535774, + "pass@32": 0.9966668378770552, + "pass@64": 0.998258668637522, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9605546875, + "pass@2": 0.9784418061023628, + "pass@4": 0.9875334872515938, + "pass@8": 0.9929514611024697, + "pass@16": 0.9961726755133811, + "pass@32": 0.99805076966085, + "pass@64": 0.9993775147500865, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9503125, + "pass@2": 0.9736109744094493, + "pass@4": 0.9853032450318711, + "pass@8": 0.9918039403469084, + "pass@16": 0.9953202736206094, + "pass@32": 0.997103633931122, + "pass@64": 0.9982579640633262, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9061171875, + "2": 0.9942578125, + "3": 0.992875, + "4": 0.9973515625, + "5": 0.9612109375, + "6": 0.9711171875, + "7": 0.974828125, + "8": 0.9605546875, + "9": 0.9503125 + }, + "per_op_avg_response_len": { + "10": 279.5555703125, + "2": 116.3177890625, + "3": 145.1886796875, + "4": 159.696953125, + "5": 175.5341328125, + "6": 200.2167890625, + "7": 224.26271875, + "8": 244.0153125, + "9": 258.968265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.5555703125, + "2": 116.3177890625, + "3": 145.1886796875, + "4": 159.696953125, + "5": 175.5341328125, + "6": 200.2167890625, + "7": 224.26271875, + "8": 244.0153125, + "9": 258.968265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 366247, + "answer_accuracy": 0.9663305260047281, + "avg_response_len": 190.81433373438028, + "resp_tokens_sum": 72320159, + "pass_at_k": { + "pass@1": 0.9663305260047281, + "pass@2": 0.9777502463122968, + "pass@4": 0.9843131039091239, + "pass@8": 0.988829560613863, + "pass@16": 0.9920626565777462, + "pass@32": 0.9943957396394841, + "pass@64": 0.9961721165046875, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9146155973451328, + "pass@2": 0.9324324814763199, + "pass@4": 0.9445238803114209, + "pass@8": 0.9560802593836489, + "pass@16": 0.96734016035308, + "pass@32": 0.9768881715308431, + "pass@64": 0.9837203706103511, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.994304906542056, + "pass@2": 0.9964129228297398, + "pass@4": 0.9970288185355336, + "pass@8": 0.9972629384829463, + "pass@16": 0.9976175582976479, + "pass@32": 0.998252262859666, + "pass@64": 0.9992273162116417, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9920441513761468, + "pass@2": 0.9957574827229165, + "pass@4": 0.9982279637637039, + "pass@8": 0.9995162799671178, + "pass@16": 0.999947230986137, + "pass@32": 0.9999995668020062, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.9982898622047245, + "pass@4": 0.9995612441999437, + "pass@8": 0.9999643954713182, + "pass@16": 0.9999997738252621, + "pass@32": 0.999999999998108, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.958953373015873, + "pass@2": 0.9680770372453444, + "pass@4": 0.9740893447842829, + "pass@8": 0.9789096637141826, + "pass@16": 0.9831310800121967, + "pass@32": 0.9872438000394951, + "pass@64": 0.9908459423510207, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9576627994011976, + "pass@2": 0.9742090621905796, + "pass@4": 0.9835539713598672, + "pass@8": 0.9882820716427391, + "pass@16": 0.990724875582214, + "pass@32": 0.9923084077866816, + "pass@64": 0.9940076504085662, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9728873890532544, + "pass@2": 0.9825437235009084, + "pass@4": 0.9887106077346841, + "pass@8": 0.9936818827626992, + "pass@16": 0.9969168428360782, + "pass@32": 0.9986419811280367, + "pass@64": 0.999638422528372, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.962715344551282, + "pass@2": 0.9790779420300828, + "pass@4": 0.9875604915371156, + "pass@8": 0.9936238350606069, + "pass@16": 0.9972435749562748, + "pass@32": 0.9989330066453073, + "pass@64": 0.9998089061705232, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9517605633802817, + "pass@2": 0.9752328934235335, + "pass@4": 0.9869940377171162, + "pass@8": 0.9930800529074789, + "pass@16": 0.9962394830525667, + "pass@32": 0.997692865617325, + "pass@64": 0.998589493557562, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 378445, + "answer_accuracy": 0.9687423206094364, + "avg_response_len": 198.46175663499344, + "resp_tokens_sum": 77530276, + "pass_at_k": { + "pass@1": 0.9687423206094364, + "pass@2": 0.9798194887694668, + "pass@4": 0.9858888134969379, + "pass@8": 0.9896003164595581, + "pass@16": 0.9920821153307023, + "pass@32": 0.9939596592617876, + "pass@64": 0.9956381403036101, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9029907975460123, + "pass@2": 0.9253232784648083, + "pass@4": 0.9409250858596658, + "pass@8": 0.9524413143135337, + "pass@16": 0.9606720946811212, + "pass@32": 0.9670849168030893, + "pass@64": 0.9737034329455921, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.9953219984802432, + "pass@2": 0.9966646871933561, + "pass@4": 0.9974002140309969, + "pass@8": 0.9979503307874706, + "pass@16": 0.9986592398659601, + "pass@32": 0.9994804612838031, + "pass@64": 0.9999579826782162, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9909801136363636, + "pass@2": 0.9946910045335242, + "pass@4": 0.9970738387815159, + "pass@8": 0.9986093258323127, + "pass@16": 0.9995114513883049, + "pass@32": 0.9999422259297166, + "pass@64": 0.9999998119654753, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996925850591716, + "pass@2": 0.9995042340306572, + "pass@4": 0.9999693070910515, + "pass@8": 0.9999998018014863, + "pass@16": 0.9999999999967328, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9676630434782608, + "pass@2": 0.9740874272509417, + "pass@4": 0.9785426351597355, + "pass@8": 0.9824365515513749, + "pass@16": 0.9864771067978614, + "pass@32": 0.9908202891990001, + "pass@64": 0.9951658357830683, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770997492836676, + "pass@2": 0.9875339834848725, + "pass@4": 0.9922179205572083, + "pass@8": 0.9941698373402024, + "pass@16": 0.9956908346337774, + "pass@32": 0.996835945780172, + "pass@64": 0.997129655270761, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.980152637283237, + "pass@2": 0.9894811626689727, + "pass@4": 0.9935971754433873, + "pass@8": 0.9961247492110364, + "pass@16": 0.99730971629554, + "pass@32": 0.9978286491723424, + "pass@64": 0.9985549131017292, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9586487676056338, + "pass@2": 0.978224464899634, + "pass@4": 0.9877859611738672, + "pass@8": 0.9920112997099682, + "pass@16": 0.9946455834727146, + "pass@32": 0.9966832051144756, + "pass@64": 0.9986007521656389, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9475346182634731, + "pass@2": 0.9713498691593192, + "pass@4": 0.9841093277449593, + "pass@8": 0.991522664265585, + "pass@16": 0.9947868178047816, + "pass@32": 0.9960634598496626, + "pass@64": 0.9968276671555734, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 370012, + "answer_accuracy": 0.9677665718111818, + "avg_response_len": 211.93494727150988, + "resp_tokens_sum": 81030360, + "pass_at_k": { + "pass@1": 0.9677665718111818, + "pass@2": 0.9793915036022247, + "pass@4": 0.9863721041084299, + "pass@8": 0.9911894920534978, + "pass@16": 0.9942968200608154, + "pass@32": 0.9961915430835045, + "pass@64": 0.9975506947543168, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9005597014925373, + "pass@2": 0.931050946057116, + "pass@4": 0.9538955333008745, + "pass@8": 0.9713410311545484, + "pass@16": 0.9821343375480532, + "pass@32": 0.9878797954047336, + "pass@64": 0.9913859266157925, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9932142857142857, + "pass@2": 0.9958703599550055, + "pass@4": 0.9975555758208795, + "pass@8": 0.9988223052875971, + "pass@16": 0.9996540969253741, + "pass@32": 0.9999717161726899, + "pass@64": 0.9999999655157344, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9954901603498543, + "pass@2": 0.998288678519777, + "pass@4": 0.9996947698410874, + "pass@8": 0.9999878398913509, + "pass@16": 0.9999999795348873, + "pass@32": 0.9999999999999929, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998204891329374, + "pass@4": 0.9999901437758876, + "pass@8": 0.9999999784883499, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9567555147058824, + "pass@2": 0.9696738217924962, + "pass@4": 0.9785373726262158, + "pass@8": 0.9853053048265434, + "pass@16": 0.9899959490701858, + "pass@32": 0.9924198270729687, + "pass@64": 0.9941155967279394, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9787066246056783, + "pass@2": 0.9856130772001291, + "pass@4": 0.9893927832167666, + "pass@8": 0.9920141237962015, + "pass@16": 0.993923739223236, + "pass@32": 0.9955704271400222, + "pass@64": 0.9966477762794999, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9710739715189873, + "pass@2": 0.9804854916276287, + "pass@4": 0.9853905750902974, + "pass@8": 0.9884781424038828, + "pass@16": 0.9907223592934364, + "pass@32": 0.993282074816925, + "pass@64": 0.9964584869928289, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9605621246246246, + "pass@2": 0.9780774869357546, + "pass@4": 0.9872390320579295, + "pass@8": 0.9933237632628275, + "pass@16": 0.9967972912739032, + "pass@32": 0.9986820833990249, + "pass@64": 0.9998014084266714, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.95164288585209, + "pass@2": 0.9741879130060515, + "pass@4": 0.984655453940444, + "pass@8": 0.9906493623795749, + "pass@16": 0.9948439227979123, + "pass@32": 0.9975481384153831, + "pass@64": 0.999415604567299, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.520346875, + "avg_response_len": 362.45493828125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.520346875, + "pass@2": 0.6050519562007803, + "pass@4": 0.6740247124671819, + "pass@8": 0.7286853796929692, + "pass@16": 0.7725788861573003, + "pass@32": 0.8091521807230214, + "pass@64": 0.8404442524344273, + "pass@128": 0.8676 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.898015625, + "pass@2": 0.935235236220473, + "pass@4": 0.9575128002437192, + "pass@8": 0.9704416052342938, + "pass@16": 0.9780518204584171, + "pass@32": 0.9836632331764502, + "pass@64": 0.9883294538000471, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.8461640625, + "pass@2": 0.8927360974409457, + "pass@4": 0.9184761690101231, + "pass@8": 0.9344069302626675, + "pass@16": 0.9452682084971497, + "pass@32": 0.9536914403793331, + "pass@64": 0.9612535993215502, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.714515625, + "pass@2": 0.7950198080708663, + "pass@4": 0.8481068088676422, + "pass@8": 0.8819388480812936, + "pass@16": 0.9045033440743804, + "pass@32": 0.922642688310152, + "pass@64": 0.9391500934773205, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.6444140625, + "pass@2": 0.7367026328740154, + "pass@4": 0.8048043104611924, + "pass@8": 0.8539345938648281, + "pass@16": 0.8889069995622597, + "pass@32": 0.9135038434620463, + "pass@64": 0.930176155233334, + "pass@128": 0.942 + }, + "15": { + "pass@1": 0.5660078125, + "pass@2": 0.6673650344488189, + "pass@4": 0.7425340289651288, + "pass@8": 0.7972093709177968, + "pass@16": 0.8378144130557783, + "pass@32": 0.869323322701449, + "pass@64": 0.8946280248830312, + "pass@128": 0.915 + }, + "16": { + "pass@1": 0.4358359375, + "pass@2": 0.5450682824803151, + "pass@4": 0.6337557572178482, + "pass@8": 0.7030683280696618, + "pass@16": 0.7588555149219782, + "pass@32": 0.804970658433809, + "pass@64": 0.8437043764872961, + "pass@128": 0.878 + }, + "17": { + "pass@1": 0.352796875, + "pass@2": 0.45748499015748095, + "pass@4": 0.5484747998687666, + "pass@8": 0.6238846178410963, + "pass@16": 0.6868115184435492, + "pass@32": 0.7409492322957991, + "pass@64": 0.7885017632595638, + "pass@128": 0.828 + }, + "18": { + "pass@1": 0.3243125, + "pass@2": 0.4265187007874019, + "pass@4": 0.5188428872328458, + "pass@8": 0.5950043644707422, + "pass@16": 0.6558389506915384, + "pass@32": 0.7055650371132244, + "pass@64": 0.7489407177736084, + "pass@128": 0.788 + }, + "19": { + "pass@1": 0.2116796875, + "pass@2": 0.2959559547244097, + "pass@4": 0.3810210404011996, + "pass@8": 0.46053649714670725, + "pass@16": 0.5328893677720297, + "pass@32": 0.5978933635265434, + "pass@64": 0.6555335144945967, + "pass@128": 0.707 + }, + "20": { + "pass@1": 0.2097265625, + "pass@2": 0.2984328248031499, + "pass@4": 0.38671852240344934, + "pass@8": 0.46642864104064785, + "pass@16": 0.5368487240957533, + "pass@32": 0.5993189878313755, + "pass@64": 0.6542248256138411, + "pass@128": 0.703 + } + }, + "per_op_accuracy": { + "11": 0.898015625, + "12": 0.8461640625, + "13": 0.714515625, + "14": 0.6444140625, + "15": 0.5660078125, + "16": 0.4358359375, + "17": 0.352796875, + "18": 0.3243125, + "19": 0.2116796875, + "20": 0.2097265625 + }, + "per_op_avg_response_len": { + "11": 297.9748125, + "12": 308.2818984375, + "13": 326.6051171875, + "14": 350.120671875, + "15": 366.8876328125, + "16": 372.691015625, + "17": 390.44553125, + "18": 400.5127734375, + "19": 398.733921875, + "20": 412.2960078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.9748125, + "12": 308.2818984375, + "13": 326.6051171875, + "14": 350.120671875, + "15": 366.8876328125, + "16": 372.691015625, + "17": 390.44553125, + "18": 400.5127734375, + "19": 398.733921875, + "20": 412.2960078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 228375, + "answer_accuracy": 0.5302168462109955, + "avg_response_len": 344.37604940564637, + "resp_tokens_sum": 148329652, + "pass_at_k": { + "pass@1": 0.5302168462109955, + "pass@2": 0.6143674828304353, + "pass@4": 0.6814863015637193, + "pass@8": 0.7332695530253647, + "pass@16": 0.7739992169546921, + "pass@32": 0.8075443740063188, + "pass@64": 0.8362693503581762, + "pass@128": 0.861218424962853 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8837420325779037, + "pass@2": 0.9222653548883589, + "pass@4": 0.947920507015235, + "pass@8": 0.9637010279911048, + "pass@16": 0.9734611724181728, + "pass@32": 0.9806739735564488, + "pass@64": 0.986230616931157, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8454051906779662, + "pass@2": 0.8960714027536811, + "pass@4": 0.9262268214884147, + "pass@8": 0.9453804812738749, + "pass@16": 0.9581318150251278, + "pass@32": 0.967175994628253, + "pass@64": 0.9743604584017613, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7345219435736677, + "pass@2": 0.8140481141855702, + "pass@4": 0.8653478248604496, + "pass@8": 0.8982865418367988, + "pass@16": 0.9215964864319973, + "pass@32": 0.9410137221597303, + "pass@64": 0.9567587524332927, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6530860591900312, + "pass@2": 0.7375094285819406, + "pass@4": 0.7984980262397106, + "pass@8": 0.8436866437249195, + "pass@16": 0.877439694440112, + "pass@32": 0.9021207551468892, + "pass@64": 0.9209834187791976, + "pass@128": 0.9376947040498442 + }, + "15": { + "pass@1": 0.5775830924855492, + "pass@2": 0.6790915439670473, + "pass@4": 0.7519301693140955, + "pass@8": 0.8032143561437084, + "pass@16": 0.8414897820739647, + "pass@32": 0.8718474109587219, + "pass@64": 0.8954041203257107, + "pass@128": 0.9132947976878613 + }, + "16": { + "pass@1": 0.44700100806451615, + "pass@2": 0.5608863030226062, + "pass@4": 0.65082626032633, + "pass@8": 0.7172614566129137, + "pass@16": 0.7691352922935121, + "pass@32": 0.811191202357312, + "pass@64": 0.8450055135573672, + "pass@128": 0.8709677419354839 + }, + "17": { + "pass@1": 0.36517792630057805, + "pass@2": 0.4668515457876293, + "pass@4": 0.5531280903832098, + "pass@8": 0.6243796018336796, + "pass@16": 0.6856354062343508, + "pass@32": 0.7387477271185189, + "pass@64": 0.7864774150053151, + "pass@128": 0.8265895953757225 + }, + "18": { + "pass@1": 0.32254464285714285, + "pass@2": 0.42149184476940393, + "pass@4": 0.5080175395039904, + "pass@8": 0.5764894560738196, + "pass@16": 0.6300392523562295, + "pass@32": 0.6761651636864748, + "pass@64": 0.720521654624119, + "pass@128": 0.7628571428571429 + }, + "19": { + "pass@1": 0.22046265015015015, + "pass@2": 0.302883961717623, + "pass@4": 0.38400145280263365, + "pass@8": 0.4581139242422052, + "pass@16": 0.5229660657681785, + "pass@32": 0.5785577026141271, + "pass@64": 0.6270164091987109, + "pass@128": 0.6726726726726727 + }, + "20": { + "pass@1": 0.23399962462462462, + "pass@2": 0.3285351049868764, + "pass@4": 0.41847575753481286, + "pass@8": 0.4943010984630937, + "pass@16": 0.5542666474227655, + "pass@32": 0.603638892582803, + "pass@64": 0.6464860980212646, + "pass@128": 0.6846846846846847 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 221547, + "answer_accuracy": 0.5146702163247101, + "avg_response_len": 357.8955615336009, + "resp_tokens_sum": 154061155, + "pass_at_k": { + "pass@1": 0.5146702163247101, + "pass@2": 0.5995868439198236, + "pass@4": 0.6694281044653121, + "pass@8": 0.7252252441277945, + "pass@16": 0.7702228210906457, + "pass@32": 0.8075542078141487, + "pass@64": 0.8391025036304598, + "pass@128": 0.8676776687481416 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9066500376506024, + "pass@2": 0.9419257986671091, + "pass@4": 0.9611228657600027, + "pass@8": 0.9717757165157694, + "pass@16": 0.9779676215597728, + "pass@32": 0.9824873734427165, + "pass@64": 0.9870350871835528, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8398804858934169, + "pass@2": 0.879727263470985, + "pass@4": 0.9022113868799307, + "pass@8": 0.9180363645945224, + "pass@16": 0.9295941994446081, + "pass@32": 0.9391730917365382, + "pass@64": 0.9482514725157508, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6983981092436975, + "pass@2": 0.7752910711528703, + "pass@4": 0.8268174419374047, + "pass@8": 0.8626439130483324, + "pass@16": 0.8890319927667336, + "pass@32": 0.9090891021837507, + "pass@64": 0.925383852262826, + "pass@128": 0.9411764705882353 + }, + "14": { + "pass@1": 0.638797514619883, + "pass@2": 0.7361791022010403, + "pass@4": 0.8073975169222257, + "pass@8": 0.8553740123724776, + "pass@16": 0.8885118283144322, + "pass@32": 0.9124250276057662, + "pass@64": 0.9279473539616667, + "pass@128": 0.9385964912280702 + }, + "15": { + "pass@1": 0.5565577651515151, + "pass@2": 0.6616037341923169, + "pass@4": 0.7388842732726583, + "pass@8": 0.793966378770861, + "pass@16": 0.8337259313936067, + "pass@32": 0.8650791082550073, + "pass@64": 0.8931229492545467, + "pass@128": 0.9181818181818182 + }, + "16": { + "pass@1": 0.42723913817663817, + "pass@2": 0.5301973972003495, + "pass@4": 0.6149158345057298, + "pass@8": 0.6860427691873833, + "pass@16": 0.7470239971947363, + "pass@32": 0.7977074492797739, + "pass@64": 0.837881259976064, + "pass@128": 0.8746438746438746 + }, + "17": { + "pass@1": 0.3642760093167702, + "pass@2": 0.474033858879053, + "pass@4": 0.5662863288245799, + "pass@8": 0.6384272461815446, + "pass@16": 0.6951978186297798, + "pass@32": 0.7441188503447789, + "pass@64": 0.7887809088931667, + "pass@128": 0.8291925465838509 + }, + "18": { + "pass@1": 0.3282169117647059, + "pass@2": 0.4380228838582672, + "pass@4": 0.540655425516663, + "pass@8": 0.6253038269220733, + "pass@16": 0.6904040193395536, + "pass@32": 0.7409393738584734, + "pass@64": 0.7816992181683576, + "pass@128": 0.8176470588235294 + }, + "19": { + "pass@1": 0.2027529761904762, + "pass@2": 0.2866637519919384, + "pass@4": 0.3734934517113932, + "pass@8": 0.45423242964508603, + "pass@16": 0.5267002530674558, + "pass@32": 0.5911339842264164, + "pass@64": 0.6474984856874543, + "pass@128": 0.6994047619047619 + }, + "20": { + "pass@1": 0.19002619760479042, + "pass@2": 0.2760233715639587, + "pass@4": 0.3645407290031859, + "pass@8": 0.44712179898012794, + "pass@16": 0.5235950630711014, + "pass@32": 0.5922470409190072, + "pass@64": 0.6522222969510175, + "pass@128": 0.7095808383233533 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 216122, + "answer_accuracy": 0.5160309061735942, + "avg_response_len": 385.73386403575796, + "resp_tokens_sum": 161551514, + "pass_at_k": { + "pass@1": 0.5160309061735942, + "pass@2": 0.6010887610577964, + "pass@4": 0.6710754904013393, + "pass@8": 0.7275272784222685, + "pass@16": 0.773539776648077, + "pass@32": 0.8124481014119566, + "pass@64": 0.8461168828483018, + "pass@128": 0.8740831295843521 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9049107142857142, + "pass@2": 0.942718097737783, + "pass@4": 0.9644573645556207, + "pass@8": 0.976589220540312, + "pass@16": 0.9832850039268494, + "pass@32": 0.9882524207239414, + "pass@64": 0.9920457050108219, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8531154434250765, + "pass@2": 0.9018159749813379, + "pass@4": 0.9259524213716399, + "pass@8": 0.9384973687647191, + "pass@16": 0.9466330163774461, + "pass@32": 0.9532566422537484, + "pass@64": 0.9597485544795151, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.7125771604938271, + "pass@2": 0.7980232939632549, + "pass@4": 0.8545895863132843, + "pass@8": 0.8871035934478441, + "pass@16": 0.9047211218668199, + "pass@32": 0.9194892020419742, + "pass@64": 0.9369815314607124, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.641853672106825, + "pass@2": 0.736465440197201, + "pass@4": 0.8081795051953817, + "pass@8": 0.8622352195778664, + "pass@16": 0.9002308972209166, + "pass@32": 0.925441310441314, + "pass@64": 0.9411943108318142, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.5632716049382716, + "pass@2": 0.6607102714591229, + "pass@4": 0.7362172228471439, + "pass@8": 0.7940996873385447, + "pass@16": 0.8380536762293106, + "pass@32": 0.8709506567456127, + "pass@64": 0.8953321790010337, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4345271017699115, + "pass@2": 0.5460007142359413, + "pass@4": 0.6376523852660009, + "pass@8": 0.707717594498196, + "pass@16": 0.7617054610490777, + "pass@32": 0.8068025693387675, + "pass@64": 0.8485437906575631, + "pass@128": 0.887905604719764 + }, + "17": { + "pass@1": 0.32876035391566266, + "pass@2": 0.4316730504695946, + "pass@4": 0.5263502491405438, + "pass@8": 0.6092641636632107, + "pass@16": 0.6799035249628819, + "pass@32": 0.7401694244029334, + "pass@64": 0.7903407379642275, + "pass@128": 0.8283132530120482 + }, + "18": { + "pass@1": 0.32202620967741935, + "pass@2": 0.4195766922783844, + "pass@4": 0.5071416249380115, + "pass@8": 0.5826766248109669, + "pass@16": 0.6470575670690685, + "pass@32": 0.6999607829389606, + "pass@64": 0.7450980789610459, + "pass@128": 0.7838709677419354 + }, + "19": { + "pass@1": 0.21190521148036254, + "pass@2": 0.2984186549944095, + "pass@4": 0.3856639179543644, + "pass@8": 0.469373003061341, + "pass@16": 0.5491552351678592, + "pass@32": 0.6242073409545712, + "pass@64": 0.6923793324484625, + "pass@128": 0.7492447129909365 + }, + "20": { + "pass@1": 0.20521302552552553, + "pass@2": 0.2908072935139864, + "pass@4": 0.377205680652531, + "pass@8": 0.4579210041834061, + "pass@16": 0.5327242625772535, + "pass@32": 0.6020922670701319, + "pass@64": 0.6639720954688267, + "pass@128": 0.7147147147147147 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7322154605263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7322154605263158, + "pass@2": 0.7821836601222334, + "pass@4": 0.8215802294268902, + "pass@8": 0.8524063844476826, + "pass@16": 0.8768992423795055, + "pass@32": 0.8971114006084395, + "pass@64": 0.9143410363553892, + "pass@128": 0.9293157894736842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9061171875, + "pass@2": 0.9296520669291347, + "pass@4": 0.946490177071616, + "pass@8": 0.9600063218340427, + "pass@16": 0.9701224203043363, + "pass@32": 0.9773745044873484, + "pass@64": 0.9830228101934626, + "pass@128": 0.988 + }, + "2": { + "pass@1": 0.9942578125, + "pass@2": 0.9963058562992128, + "pass@4": 0.9973353727034121, + "pass@8": 0.9980348689327625, + "pass@16": 0.9986730600533266, + "pass@32": 0.9992581488007652, + "pass@64": 0.9997381327355772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.992875, + "pass@2": 0.99627374507874, + "pass@4": 0.9983502170041246, + "pass@8": 0.9993787301566442, + "pass@16": 0.9998215164710738, + "pass@32": 0.99998079290106, + "pass@64": 0.9999999379483816, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9973515625, + "pass@2": 0.9992237942913387, + "pass@4": 0.999845853112111, + "pass@8": 0.9999885322027398, + "pass@16": 0.9999999276229732, + "pass@32": 0.9999999999993945, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9612109375, + "pass@2": 0.9706935285433075, + "pass@4": 0.9771380594300711, + "pass@8": 0.9823009579962168, + "pass@16": 0.9866195147329673, + "pass@32": 0.9902375379909053, + "pass@64": 0.9934479880732294, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9711171875, + "pass@2": 0.9824745324803148, + "pass@4": 0.9884285929883763, + "pass@8": 0.9915199624038011, + "pass@16": 0.9934720350654136, + "pass@32": 0.9949225786814185, + "pass@64": 0.9959341500065584, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.974828125, + "pass@2": 0.9842936761811031, + "pass@4": 0.989352229846269, + "pass@8": 0.9928827326004378, + "pass@16": 0.9950953202535774, + "pass@32": 0.9966668378770552, + "pass@64": 0.998258668637522, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9605546875, + "pass@2": 0.9784418061023628, + "pass@4": 0.9875334872515938, + "pass@8": 0.9929514611024697, + "pass@16": 0.9961726755133811, + "pass@32": 0.99805076966085, + "pass@64": 0.9993775147500865, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9503125, + "pass@2": 0.9736109744094493, + "pass@4": 0.9853032450318711, + "pass@8": 0.9918039403469084, + "pass@16": 0.9953202736206094, + "pass@32": 0.997103633931122, + "pass@64": 0.9982579640633262, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.898015625, + "pass@2": 0.935235236220473, + "pass@4": 0.9575128002437192, + "pass@8": 0.9704416052342938, + "pass@16": 0.9780518204584171, + "pass@32": 0.9836632331764502, + "pass@64": 0.9883294538000471, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.8461640625, + "pass@2": 0.8927360974409457, + "pass@4": 0.9184761690101231, + "pass@8": 0.9344069302626675, + "pass@16": 0.9452682084971497, + "pass@32": 0.9536914403793331, + "pass@64": 0.9612535993215502, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.714515625, + "pass@2": 0.7950198080708663, + "pass@4": 0.8481068088676422, + "pass@8": 0.8819388480812936, + "pass@16": 0.9045033440743804, + "pass@32": 0.922642688310152, + "pass@64": 0.9391500934773205, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.6444140625, + "pass@2": 0.7367026328740154, + "pass@4": 0.8048043104611924, + "pass@8": 0.8539345938648281, + "pass@16": 0.8889069995622597, + "pass@32": 0.9135038434620463, + "pass@64": 0.930176155233334, + "pass@128": 0.942 + }, + "15": { + "pass@1": 0.5660078125, + "pass@2": 0.6673650344488189, + "pass@4": 0.7425340289651288, + "pass@8": 0.7972093709177968, + "pass@16": 0.8378144130557783, + "pass@32": 0.869323322701449, + "pass@64": 0.8946280248830312, + "pass@128": 0.915 + }, + "16": { + "pass@1": 0.4358359375, + "pass@2": 0.5450682824803151, + "pass@4": 0.6337557572178482, + "pass@8": 0.7030683280696618, + "pass@16": 0.7588555149219782, + "pass@32": 0.804970658433809, + "pass@64": 0.8437043764872961, + "pass@128": 0.878 + }, + "17": { + "pass@1": 0.352796875, + "pass@2": 0.45748499015748095, + "pass@4": 0.5484747998687666, + "pass@8": 0.6238846178410963, + "pass@16": 0.6868115184435492, + "pass@32": 0.7409492322957991, + "pass@64": 0.7885017632595638, + "pass@128": 0.828 + }, + "18": { + "pass@1": 0.3243125, + "pass@2": 0.4265187007874019, + "pass@4": 0.5188428872328458, + "pass@8": 0.5950043644707422, + "pass@16": 0.6558389506915384, + "pass@32": 0.7055650371132244, + "pass@64": 0.7489407177736084, + "pass@128": 0.788 + }, + "19": { + "pass@1": 0.2116796875, + "pass@2": 0.2959559547244097, + "pass@4": 0.3810210404011996, + "pass@8": 0.46053649714670725, + "pass@16": 0.5328893677720297, + "pass@32": 0.5978933635265434, + "pass@64": 0.6555335144945967, + "pass@128": 0.707 + }, + "20": { + "pass@1": 0.2097265625, + "pass@2": 0.2984328248031499, + "pass@4": 0.38671852240344934, + "pass@8": 0.46642864104064785, + "pass@16": 0.5368487240957533, + "pass@32": 0.5993189878313755, + "pass@64": 0.6542248256138411, + "pass@128": 0.703 + } + }, + "per_op_accuracy": { + "10": 0.9061171875, + "2": 0.9942578125, + "3": 0.992875, + "4": 0.9973515625, + "5": 0.9612109375, + "6": 0.9711171875, + "7": 0.974828125, + "8": 0.9605546875, + "9": 0.9503125, + "11": 0.898015625, + "12": 0.8461640625, + "13": 0.714515625, + "14": 0.6444140625, + "15": 0.5660078125, + "16": 0.4358359375, + "17": 0.352796875, + "18": 0.3243125, + "19": 0.2116796875, + "20": 0.2097265625 + }, + "per_op_avg_response_len": { + "10": 279.5555703125, + "2": 116.3177890625, + "3": 145.1886796875, + "4": 159.696953125, + "5": 175.5341328125, + "6": 200.2167890625, + "7": 224.26271875, + "8": 244.0153125, + "9": 258.968265625, + "11": 297.9748125, + "12": 308.2818984375, + "13": 326.6051171875, + "14": 350.120671875, + "15": 366.8876328125, + "16": 372.691015625, + "17": 390.44553125, + "18": 400.5127734375, + "19": 398.733921875, + "20": 412.2960078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.5555703125, + "2": 116.3177890625, + "3": 145.1886796875, + "4": 159.696953125, + "5": 175.5341328125, + "6": 200.2167890625, + "7": 224.26271875, + "8": 244.0153125, + "9": 258.968265625, + "11": 297.9748125, + "12": 308.2818984375, + "13": 326.6051171875, + "14": 350.120671875, + "15": 366.8876328125, + "16": 372.691015625, + "17": 390.44553125, + "18": 400.5127734375, + "19": 398.733921875, + "20": 412.2960078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 594622, + "answer_accuracy": 0.7343478303825483, + "avg_response_len": 272.49867980358835, + "resp_tokens_sum": 220649811, + "pass_at_k": { + "pass@1": 0.7343478303825483, + "pass@2": 0.7844554314029564, + "pass@4": 0.8232299249821057, + "pass@8": 0.8528890886670805, + "pass@16": 0.8760677981630246, + "pass@32": 0.8950035731273772, + "pass@64": 0.9111147646104464, + "pass@128": 0.9250711349984192 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9146155973451328, + "pass@2": 0.9324324814763199, + "pass@4": 0.9445238803114209, + "pass@8": 0.9560802593836489, + "pass@16": 0.96734016035308, + "pass@32": 0.9768881715308431, + "pass@64": 0.9837203706103511, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.994304906542056, + "pass@2": 0.9964129228297398, + "pass@4": 0.9970288185355336, + "pass@8": 0.9972629384829463, + "pass@16": 0.9976175582976479, + "pass@32": 0.998252262859666, + "pass@64": 0.9992273162116417, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9920441513761468, + "pass@2": 0.9957574827229165, + "pass@4": 0.9982279637637039, + "pass@8": 0.9995162799671178, + "pass@16": 0.999947230986137, + "pass@32": 0.9999995668020062, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.9982898622047245, + "pass@4": 0.9995612441999437, + "pass@8": 0.9999643954713182, + "pass@16": 0.9999997738252621, + "pass@32": 0.999999999998108, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.958953373015873, + "pass@2": 0.9680770372453444, + "pass@4": 0.9740893447842829, + "pass@8": 0.9789096637141826, + "pass@16": 0.9831310800121967, + "pass@32": 0.9872438000394951, + "pass@64": 0.9908459423510207, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9576627994011976, + "pass@2": 0.9742090621905796, + "pass@4": 0.9835539713598672, + "pass@8": 0.9882820716427391, + "pass@16": 0.990724875582214, + "pass@32": 0.9923084077866816, + "pass@64": 0.9940076504085662, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9728873890532544, + "pass@2": 0.9825437235009084, + "pass@4": 0.9887106077346841, + "pass@8": 0.9936818827626992, + "pass@16": 0.9969168428360782, + "pass@32": 0.9986419811280367, + "pass@64": 0.999638422528372, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.962715344551282, + "pass@2": 0.9790779420300828, + "pass@4": 0.9875604915371156, + "pass@8": 0.9936238350606069, + "pass@16": 0.9972435749562748, + "pass@32": 0.9989330066453073, + "pass@64": 0.9998089061705232, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9517605633802817, + "pass@2": 0.9752328934235335, + "pass@4": 0.9869940377171162, + "pass@8": 0.9930800529074789, + "pass@16": 0.9962394830525667, + "pass@32": 0.997692865617325, + "pass@64": 0.998589493557562, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8837420325779037, + "pass@2": 0.9222653548883589, + "pass@4": 0.947920507015235, + "pass@8": 0.9637010279911048, + "pass@16": 0.9734611724181728, + "pass@32": 0.9806739735564488, + "pass@64": 0.986230616931157, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8454051906779662, + "pass@2": 0.8960714027536811, + "pass@4": 0.9262268214884147, + "pass@8": 0.9453804812738749, + "pass@16": 0.9581318150251278, + "pass@32": 0.967175994628253, + "pass@64": 0.9743604584017613, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7345219435736677, + "pass@2": 0.8140481141855702, + "pass@4": 0.8653478248604496, + "pass@8": 0.8982865418367988, + "pass@16": 0.9215964864319973, + "pass@32": 0.9410137221597303, + "pass@64": 0.9567587524332927, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6530860591900312, + "pass@2": 0.7375094285819406, + "pass@4": 0.7984980262397106, + "pass@8": 0.8436866437249195, + "pass@16": 0.877439694440112, + "pass@32": 0.9021207551468892, + "pass@64": 0.9209834187791976, + "pass@128": 0.9376947040498442 + }, + "15": { + "pass@1": 0.5775830924855492, + "pass@2": 0.6790915439670473, + "pass@4": 0.7519301693140955, + "pass@8": 0.8032143561437084, + "pass@16": 0.8414897820739647, + "pass@32": 0.8718474109587219, + "pass@64": 0.8954041203257107, + "pass@128": 0.9132947976878613 + }, + "16": { + "pass@1": 0.44700100806451615, + "pass@2": 0.5608863030226062, + "pass@4": 0.65082626032633, + "pass@8": 0.7172614566129137, + "pass@16": 0.7691352922935121, + "pass@32": 0.811191202357312, + "pass@64": 0.8450055135573672, + "pass@128": 0.8709677419354839 + }, + "17": { + "pass@1": 0.36517792630057805, + "pass@2": 0.4668515457876293, + "pass@4": 0.5531280903832098, + "pass@8": 0.6243796018336796, + "pass@16": 0.6856354062343508, + "pass@32": 0.7387477271185189, + "pass@64": 0.7864774150053151, + "pass@128": 0.8265895953757225 + }, + "18": { + "pass@1": 0.32254464285714285, + "pass@2": 0.42149184476940393, + "pass@4": 0.5080175395039904, + "pass@8": 0.5764894560738196, + "pass@16": 0.6300392523562295, + "pass@32": 0.6761651636864748, + "pass@64": 0.720521654624119, + "pass@128": 0.7628571428571429 + }, + "19": { + "pass@1": 0.22046265015015015, + "pass@2": 0.302883961717623, + "pass@4": 0.38400145280263365, + "pass@8": 0.4581139242422052, + "pass@16": 0.5229660657681785, + "pass@32": 0.5785577026141271, + "pass@64": 0.6270164091987109, + "pass@128": 0.6726726726726727 + }, + "20": { + "pass@1": 0.23399962462462462, + "pass@2": 0.3285351049868764, + "pass@4": 0.41847575753481286, + "pass@8": 0.4943010984630937, + "pass@16": 0.5542666474227655, + "pass@32": 0.603638892582803, + "pass@64": 0.6464860980212646, + "pass@128": 0.6846846846846847 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 599992, + "answer_accuracy": 0.7306995323460639, + "avg_response_len": 282.0433444563523, + "resp_tokens_sum": 231591431, + "pass_at_k": { + "pass@1": 0.7306995323460639, + "pass@2": 0.7804863033245112, + "pass@4": 0.8199874316616441, + "pass@8": 0.8510043120555448, + "pass@16": 0.8757745850845181, + "pass@32": 0.8962384537717814, + "pass@64": 0.9135758883734845, + "pass@128": 0.9293842556508184 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9029907975460123, + "pass@2": 0.9253232784648083, + "pass@4": 0.9409250858596658, + "pass@8": 0.9524413143135337, + "pass@16": 0.9606720946811212, + "pass@32": 0.9670849168030893, + "pass@64": 0.9737034329455921, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.9953219984802432, + "pass@2": 0.9966646871933561, + "pass@4": 0.9974002140309969, + "pass@8": 0.9979503307874706, + "pass@16": 0.9986592398659601, + "pass@32": 0.9994804612838031, + "pass@64": 0.9999579826782162, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9909801136363636, + "pass@2": 0.9946910045335242, + "pass@4": 0.9970738387815159, + "pass@8": 0.9986093258323127, + "pass@16": 0.9995114513883049, + "pass@32": 0.9999422259297166, + "pass@64": 0.9999998119654753, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996925850591716, + "pass@2": 0.9995042340306572, + "pass@4": 0.9999693070910515, + "pass@8": 0.9999998018014863, + "pass@16": 0.9999999999967328, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9676630434782608, + "pass@2": 0.9740874272509417, + "pass@4": 0.9785426351597355, + "pass@8": 0.9824365515513749, + "pass@16": 0.9864771067978614, + "pass@32": 0.9908202891990001, + "pass@64": 0.9951658357830683, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770997492836676, + "pass@2": 0.9875339834848725, + "pass@4": 0.9922179205572083, + "pass@8": 0.9941698373402024, + "pass@16": 0.9956908346337774, + "pass@32": 0.996835945780172, + "pass@64": 0.997129655270761, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.980152637283237, + "pass@2": 0.9894811626689727, + "pass@4": 0.9935971754433873, + "pass@8": 0.9961247492110364, + "pass@16": 0.99730971629554, + "pass@32": 0.9978286491723424, + "pass@64": 0.9985549131017292, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9586487676056338, + "pass@2": 0.978224464899634, + "pass@4": 0.9877859611738672, + "pass@8": 0.9920112997099682, + "pass@16": 0.9946455834727146, + "pass@32": 0.9966832051144756, + "pass@64": 0.9986007521656389, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9475346182634731, + "pass@2": 0.9713498691593192, + "pass@4": 0.9841093277449593, + "pass@8": 0.991522664265585, + "pass@16": 0.9947868178047816, + "pass@32": 0.9960634598496626, + "pass@64": 0.9968276671555734, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9066500376506024, + "pass@2": 0.9419257986671091, + "pass@4": 0.9611228657600027, + "pass@8": 0.9717757165157694, + "pass@16": 0.9779676215597728, + "pass@32": 0.9824873734427165, + "pass@64": 0.9870350871835528, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8398804858934169, + "pass@2": 0.879727263470985, + "pass@4": 0.9022113868799307, + "pass@8": 0.9180363645945224, + "pass@16": 0.9295941994446081, + "pass@32": 0.9391730917365382, + "pass@64": 0.9482514725157508, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6983981092436975, + "pass@2": 0.7752910711528703, + "pass@4": 0.8268174419374047, + "pass@8": 0.8626439130483324, + "pass@16": 0.8890319927667336, + "pass@32": 0.9090891021837507, + "pass@64": 0.925383852262826, + "pass@128": 0.9411764705882353 + }, + "14": { + "pass@1": 0.638797514619883, + "pass@2": 0.7361791022010403, + "pass@4": 0.8073975169222257, + "pass@8": 0.8553740123724776, + "pass@16": 0.8885118283144322, + "pass@32": 0.9124250276057662, + "pass@64": 0.9279473539616667, + "pass@128": 0.9385964912280702 + }, + "15": { + "pass@1": 0.5565577651515151, + "pass@2": 0.6616037341923169, + "pass@4": 0.7388842732726583, + "pass@8": 0.793966378770861, + "pass@16": 0.8337259313936067, + "pass@32": 0.8650791082550073, + "pass@64": 0.8931229492545467, + "pass@128": 0.9181818181818182 + }, + "16": { + "pass@1": 0.42723913817663817, + "pass@2": 0.5301973972003495, + "pass@4": 0.6149158345057298, + "pass@8": 0.6860427691873833, + "pass@16": 0.7470239971947363, + "pass@32": 0.7977074492797739, + "pass@64": 0.837881259976064, + "pass@128": 0.8746438746438746 + }, + "17": { + "pass@1": 0.3642760093167702, + "pass@2": 0.474033858879053, + "pass@4": 0.5662863288245799, + "pass@8": 0.6384272461815446, + "pass@16": 0.6951978186297798, + "pass@32": 0.7441188503447789, + "pass@64": 0.7887809088931667, + "pass@128": 0.8291925465838509 + }, + "18": { + "pass@1": 0.3282169117647059, + "pass@2": 0.4380228838582672, + "pass@4": 0.540655425516663, + "pass@8": 0.6253038269220733, + "pass@16": 0.6904040193395536, + "pass@32": 0.7409393738584734, + "pass@64": 0.7816992181683576, + "pass@128": 0.8176470588235294 + }, + "19": { + "pass@1": 0.2027529761904762, + "pass@2": 0.2866637519919384, + "pass@4": 0.3734934517113932, + "pass@8": 0.45423242964508603, + "pass@16": 0.5267002530674558, + "pass@32": 0.5911339842264164, + "pass@64": 0.6474984856874543, + "pass@128": 0.6994047619047619 + }, + "20": { + "pass@1": 0.19002619760479042, + "pass@2": 0.2760233715639587, + "pass@4": 0.3645407290031859, + "pass@8": 0.44712179898012794, + "pass@16": 0.5235950630711014, + "pass@32": 0.5922470409190072, + "pass@64": 0.6522222969510175, + "pass@128": 0.7095808383233533 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 586134, + "answer_accuracy": 0.7316139758747404, + "avg_response_len": 302.79132299488737, + "resp_tokens_sum": 242581874, + "pass_at_k": { + "pass@1": 0.7316139758747404, + "pass@2": 0.7816272323759221, + "pass@4": 0.8215453713955919, + "pass@8": 0.8533555308773647, + "pass@16": 0.8788922752379291, + "pass@32": 0.9001364957677545, + "pass@64": 0.9183860626155672, + "pass@128": 0.9335357085796453 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9005597014925373, + "pass@2": 0.931050946057116, + "pass@4": 0.9538955333008745, + "pass@8": 0.9713410311545484, + "pass@16": 0.9821343375480532, + "pass@32": 0.9878797954047336, + "pass@64": 0.9913859266157925, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9932142857142857, + "pass@2": 0.9958703599550055, + "pass@4": 0.9975555758208795, + "pass@8": 0.9988223052875971, + "pass@16": 0.9996540969253741, + "pass@32": 0.9999717161726899, + "pass@64": 0.9999999655157344, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9954901603498543, + "pass@2": 0.998288678519777, + "pass@4": 0.9996947698410874, + "pass@8": 0.9999878398913509, + "pass@16": 0.9999999795348873, + "pass@32": 0.9999999999999929, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998204891329374, + "pass@4": 0.9999901437758876, + "pass@8": 0.9999999784883499, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9567555147058824, + "pass@2": 0.9696738217924962, + "pass@4": 0.9785373726262158, + "pass@8": 0.9853053048265434, + "pass@16": 0.9899959490701858, + "pass@32": 0.9924198270729687, + "pass@64": 0.9941155967279394, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9787066246056783, + "pass@2": 0.9856130772001291, + "pass@4": 0.9893927832167666, + "pass@8": 0.9920141237962015, + "pass@16": 0.993923739223236, + "pass@32": 0.9955704271400222, + "pass@64": 0.9966477762794999, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9710739715189873, + "pass@2": 0.9804854916276287, + "pass@4": 0.9853905750902974, + "pass@8": 0.9884781424038828, + "pass@16": 0.9907223592934364, + "pass@32": 0.993282074816925, + "pass@64": 0.9964584869928289, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9605621246246246, + "pass@2": 0.9780774869357546, + "pass@4": 0.9872390320579295, + "pass@8": 0.9933237632628275, + "pass@16": 0.9967972912739032, + "pass@32": 0.9986820833990249, + "pass@64": 0.9998014084266714, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.95164288585209, + "pass@2": 0.9741879130060515, + "pass@4": 0.984655453940444, + "pass@8": 0.9906493623795749, + "pass@16": 0.9948439227979123, + "pass@32": 0.9975481384153831, + "pass@64": 0.999415604567299, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9049107142857142, + "pass@2": 0.942718097737783, + "pass@4": 0.9644573645556207, + "pass@8": 0.976589220540312, + "pass@16": 0.9832850039268494, + "pass@32": 0.9882524207239414, + "pass@64": 0.9920457050108219, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8531154434250765, + "pass@2": 0.9018159749813379, + "pass@4": 0.9259524213716399, + "pass@8": 0.9384973687647191, + "pass@16": 0.9466330163774461, + "pass@32": 0.9532566422537484, + "pass@64": 0.9597485544795151, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.7125771604938271, + "pass@2": 0.7980232939632549, + "pass@4": 0.8545895863132843, + "pass@8": 0.8871035934478441, + "pass@16": 0.9047211218668199, + "pass@32": 0.9194892020419742, + "pass@64": 0.9369815314607124, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.641853672106825, + "pass@2": 0.736465440197201, + "pass@4": 0.8081795051953817, + "pass@8": 0.8622352195778664, + "pass@16": 0.9002308972209166, + "pass@32": 0.925441310441314, + "pass@64": 0.9411943108318142, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.5632716049382716, + "pass@2": 0.6607102714591229, + "pass@4": 0.7362172228471439, + "pass@8": 0.7940996873385447, + "pass@16": 0.8380536762293106, + "pass@32": 0.8709506567456127, + "pass@64": 0.8953321790010337, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4345271017699115, + "pass@2": 0.5460007142359413, + "pass@4": 0.6376523852660009, + "pass@8": 0.707717594498196, + "pass@16": 0.7617054610490777, + "pass@32": 0.8068025693387675, + "pass@64": 0.8485437906575631, + "pass@128": 0.887905604719764 + }, + "17": { + "pass@1": 0.32876035391566266, + "pass@2": 0.4316730504695946, + "pass@4": 0.5263502491405438, + "pass@8": 0.6092641636632107, + "pass@16": 0.6799035249628819, + "pass@32": 0.7401694244029334, + "pass@64": 0.7903407379642275, + "pass@128": 0.8283132530120482 + }, + "18": { + "pass@1": 0.32202620967741935, + "pass@2": 0.4195766922783844, + "pass@4": 0.5071416249380115, + "pass@8": 0.5826766248109669, + "pass@16": 0.6470575670690685, + "pass@32": 0.6999607829389606, + "pass@64": 0.7450980789610459, + "pass@128": 0.7838709677419354 + }, + "19": { + "pass@1": 0.21190521148036254, + "pass@2": 0.2984186549944095, + "pass@4": 0.3856639179543644, + "pass@8": 0.469373003061341, + "pass@16": 0.5491552351678592, + "pass@32": 0.6242073409545712, + "pass@64": 0.6923793324484625, + "pass@128": 0.7492447129909365 + }, + "20": { + "pass@1": 0.20521302552552553, + "pass@2": 0.2908072935139864, + "pass@4": 0.377205680652531, + "pass@8": 0.4579210041834061, + "pass@16": 0.5327242625772535, + "pass@32": 0.6020922670701319, + "pass@64": 0.6639720954688267, + "pass@128": 0.7147147147147147 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f2ad53aefed46e5f8cf9641d16d9342880daf8d3 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516aeadfd5ea8db4687b007224a0e312e54f0f4df7ca2fc1ac32e8f8f51949ea +size 2676663519 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/summary.csv b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9bfeaa19ad40dae89b89a097d9502c724c0d524f --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op11-14,0.967625,0.520346875,0.7322154605263158,0.0,0.0,0.0,200.41735677083332,362.45493828125,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/summary.json b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..546acda83a38115a7a80ffc695ef6d707bbdfb28 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op11-14", + "id_acc": 0.967625, + "ood_acc": 0.520346875, + "total_acc": 0.7322154605263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.41735677083332, + "ood_avg_resp_len": 362.45493828125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 594622, + "answer_accuracy": 0.7343478303825483, + "avg_response_len": 272.49867980358835, + "resp_tokens_sum": 220649811, + "pass_at_k": { + "pass@1": 0.7343478303825483, + "pass@2": 0.7844554314029564, + "pass@4": 0.8232299249821057, + "pass@8": 0.8528890886670805, + "pass@16": 0.8760677981630246, + "pass@32": 0.8950035731273772, + "pass@64": 0.9111147646104464, + "pass@128": 0.9250711349984192 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9146155973451328, + "pass@2": 0.9324324814763199, + "pass@4": 0.9445238803114209, + "pass@8": 0.9560802593836489, + "pass@16": 0.96734016035308, + "pass@32": 0.9768881715308431, + "pass@64": 0.9837203706103511, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.994304906542056, + "pass@2": 0.9964129228297398, + "pass@4": 0.9970288185355336, + "pass@8": 0.9972629384829463, + "pass@16": 0.9976175582976479, + "pass@32": 0.998252262859666, + "pass@64": 0.9992273162116417, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9920441513761468, + "pass@2": 0.9957574827229165, + "pass@4": 0.9982279637637039, + "pass@8": 0.9995162799671178, + "pass@16": 0.999947230986137, + "pass@32": 0.9999995668020062, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.9982898622047245, + "pass@4": 0.9995612441999437, + "pass@8": 0.9999643954713182, + "pass@16": 0.9999997738252621, + "pass@32": 0.999999999998108, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.958953373015873, + "pass@2": 0.9680770372453444, + "pass@4": 0.9740893447842829, + "pass@8": 0.9789096637141826, + "pass@16": 0.9831310800121967, + "pass@32": 0.9872438000394951, + "pass@64": 0.9908459423510207, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9576627994011976, + "pass@2": 0.9742090621905796, + "pass@4": 0.9835539713598672, + "pass@8": 0.9882820716427391, + "pass@16": 0.990724875582214, + "pass@32": 0.9923084077866816, + "pass@64": 0.9940076504085662, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9728873890532544, + "pass@2": 0.9825437235009084, + "pass@4": 0.9887106077346841, + "pass@8": 0.9936818827626992, + "pass@16": 0.9969168428360782, + "pass@32": 0.9986419811280367, + "pass@64": 0.999638422528372, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.962715344551282, + "pass@2": 0.9790779420300828, + "pass@4": 0.9875604915371156, + "pass@8": 0.9936238350606069, + "pass@16": 0.9972435749562748, + "pass@32": 0.9989330066453073, + "pass@64": 0.9998089061705232, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9517605633802817, + "pass@2": 0.9752328934235335, + "pass@4": 0.9869940377171162, + "pass@8": 0.9930800529074789, + "pass@16": 0.9962394830525667, + "pass@32": 0.997692865617325, + "pass@64": 0.998589493557562, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8837420325779037, + "pass@2": 0.9222653548883589, + "pass@4": 0.947920507015235, + "pass@8": 0.9637010279911048, + "pass@16": 0.9734611724181728, + "pass@32": 0.9806739735564488, + "pass@64": 0.986230616931157, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8454051906779662, + "pass@2": 0.8960714027536811, + "pass@4": 0.9262268214884147, + "pass@8": 0.9453804812738749, + "pass@16": 0.9581318150251278, + "pass@32": 0.967175994628253, + "pass@64": 0.9743604584017613, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7345219435736677, + "pass@2": 0.8140481141855702, + "pass@4": 0.8653478248604496, + "pass@8": 0.8982865418367988, + "pass@16": 0.9215964864319973, + "pass@32": 0.9410137221597303, + "pass@64": 0.9567587524332927, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6530860591900312, + "pass@2": 0.7375094285819406, + "pass@4": 0.7984980262397106, + "pass@8": 0.8436866437249195, + "pass@16": 0.877439694440112, + "pass@32": 0.9021207551468892, + "pass@64": 0.9209834187791976, + "pass@128": 0.9376947040498442 + }, + "15": { + "pass@1": 0.5775830924855492, + "pass@2": 0.6790915439670473, + "pass@4": 0.7519301693140955, + "pass@8": 0.8032143561437084, + "pass@16": 0.8414897820739647, + "pass@32": 0.8718474109587219, + "pass@64": 0.8954041203257107, + "pass@128": 0.9132947976878613 + }, + "16": { + "pass@1": 0.44700100806451615, + "pass@2": 0.5608863030226062, + "pass@4": 0.65082626032633, + "pass@8": 0.7172614566129137, + "pass@16": 0.7691352922935121, + "pass@32": 0.811191202357312, + "pass@64": 0.8450055135573672, + "pass@128": 0.8709677419354839 + }, + "17": { + "pass@1": 0.36517792630057805, + "pass@2": 0.4668515457876293, + "pass@4": 0.5531280903832098, + "pass@8": 0.6243796018336796, + "pass@16": 0.6856354062343508, + "pass@32": 0.7387477271185189, + "pass@64": 0.7864774150053151, + "pass@128": 0.8265895953757225 + }, + "18": { + "pass@1": 0.32254464285714285, + "pass@2": 0.42149184476940393, + "pass@4": 0.5080175395039904, + "pass@8": 0.5764894560738196, + "pass@16": 0.6300392523562295, + "pass@32": 0.6761651636864748, + "pass@64": 0.720521654624119, + "pass@128": 0.7628571428571429 + }, + "19": { + "pass@1": 0.22046265015015015, + "pass@2": 0.302883961717623, + "pass@4": 0.38400145280263365, + "pass@8": 0.4581139242422052, + "pass@16": 0.5229660657681785, + "pass@32": 0.5785577026141271, + "pass@64": 0.6270164091987109, + "pass@128": 0.6726726726726727 + }, + "20": { + "pass@1": 0.23399962462462462, + "pass@2": 0.3285351049868764, + "pass@4": 0.41847575753481286, + "pass@8": 0.4943010984630937, + "pass@16": 0.5542666474227655, + "pass@32": 0.603638892582803, + "pass@64": 0.6464860980212646, + "pass@128": 0.6846846846846847 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 599992, + "answer_accuracy": 0.7306995323460639, + "avg_response_len": 282.0433444563523, + "resp_tokens_sum": 231591431, + "pass_at_k": { + "pass@1": 0.7306995323460639, + "pass@2": 0.7804863033245112, + "pass@4": 0.8199874316616441, + "pass@8": 0.8510043120555448, + "pass@16": 0.8757745850845181, + "pass@32": 0.8962384537717814, + "pass@64": 0.9135758883734845, + "pass@128": 0.9293842556508184 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9029907975460123, + "pass@2": 0.9253232784648083, + "pass@4": 0.9409250858596658, + "pass@8": 0.9524413143135337, + "pass@16": 0.9606720946811212, + "pass@32": 0.9670849168030893, + "pass@64": 0.9737034329455921, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.9953219984802432, + "pass@2": 0.9966646871933561, + "pass@4": 0.9974002140309969, + "pass@8": 0.9979503307874706, + "pass@16": 0.9986592398659601, + "pass@32": 0.9994804612838031, + "pass@64": 0.9999579826782162, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9909801136363636, + "pass@2": 0.9946910045335242, + "pass@4": 0.9970738387815159, + "pass@8": 0.9986093258323127, + "pass@16": 0.9995114513883049, + "pass@32": 0.9999422259297166, + "pass@64": 0.9999998119654753, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996925850591716, + "pass@2": 0.9995042340306572, + "pass@4": 0.9999693070910515, + "pass@8": 0.9999998018014863, + "pass@16": 0.9999999999967328, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9676630434782608, + "pass@2": 0.9740874272509417, + "pass@4": 0.9785426351597355, + "pass@8": 0.9824365515513749, + "pass@16": 0.9864771067978614, + "pass@32": 0.9908202891990001, + "pass@64": 0.9951658357830683, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770997492836676, + "pass@2": 0.9875339834848725, + "pass@4": 0.9922179205572083, + "pass@8": 0.9941698373402024, + "pass@16": 0.9956908346337774, + "pass@32": 0.996835945780172, + "pass@64": 0.997129655270761, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.980152637283237, + "pass@2": 0.9894811626689727, + "pass@4": 0.9935971754433873, + "pass@8": 0.9961247492110364, + "pass@16": 0.99730971629554, + "pass@32": 0.9978286491723424, + "pass@64": 0.9985549131017292, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9586487676056338, + "pass@2": 0.978224464899634, + "pass@4": 0.9877859611738672, + "pass@8": 0.9920112997099682, + "pass@16": 0.9946455834727146, + "pass@32": 0.9966832051144756, + "pass@64": 0.9986007521656389, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9475346182634731, + "pass@2": 0.9713498691593192, + "pass@4": 0.9841093277449593, + "pass@8": 0.991522664265585, + "pass@16": 0.9947868178047816, + "pass@32": 0.9960634598496626, + "pass@64": 0.9968276671555734, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9066500376506024, + "pass@2": 0.9419257986671091, + "pass@4": 0.9611228657600027, + "pass@8": 0.9717757165157694, + "pass@16": 0.9779676215597728, + "pass@32": 0.9824873734427165, + "pass@64": 0.9870350871835528, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8398804858934169, + "pass@2": 0.879727263470985, + "pass@4": 0.9022113868799307, + "pass@8": 0.9180363645945224, + "pass@16": 0.9295941994446081, + "pass@32": 0.9391730917365382, + "pass@64": 0.9482514725157508, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6983981092436975, + "pass@2": 0.7752910711528703, + "pass@4": 0.8268174419374047, + "pass@8": 0.8626439130483324, + "pass@16": 0.8890319927667336, + "pass@32": 0.9090891021837507, + "pass@64": 0.925383852262826, + "pass@128": 0.9411764705882353 + }, + "14": { + "pass@1": 0.638797514619883, + "pass@2": 0.7361791022010403, + "pass@4": 0.8073975169222257, + "pass@8": 0.8553740123724776, + "pass@16": 0.8885118283144322, + "pass@32": 0.9124250276057662, + "pass@64": 0.9279473539616667, + "pass@128": 0.9385964912280702 + }, + "15": { + "pass@1": 0.5565577651515151, + "pass@2": 0.6616037341923169, + "pass@4": 0.7388842732726583, + "pass@8": 0.793966378770861, + "pass@16": 0.8337259313936067, + "pass@32": 0.8650791082550073, + "pass@64": 0.8931229492545467, + "pass@128": 0.9181818181818182 + }, + "16": { + "pass@1": 0.42723913817663817, + "pass@2": 0.5301973972003495, + "pass@4": 0.6149158345057298, + "pass@8": 0.6860427691873833, + "pass@16": 0.7470239971947363, + "pass@32": 0.7977074492797739, + "pass@64": 0.837881259976064, + "pass@128": 0.8746438746438746 + }, + "17": { + "pass@1": 0.3642760093167702, + "pass@2": 0.474033858879053, + "pass@4": 0.5662863288245799, + "pass@8": 0.6384272461815446, + "pass@16": 0.6951978186297798, + "pass@32": 0.7441188503447789, + "pass@64": 0.7887809088931667, + "pass@128": 0.8291925465838509 + }, + "18": { + "pass@1": 0.3282169117647059, + "pass@2": 0.4380228838582672, + "pass@4": 0.540655425516663, + "pass@8": 0.6253038269220733, + "pass@16": 0.6904040193395536, + "pass@32": 0.7409393738584734, + "pass@64": 0.7816992181683576, + "pass@128": 0.8176470588235294 + }, + "19": { + "pass@1": 0.2027529761904762, + "pass@2": 0.2866637519919384, + "pass@4": 0.3734934517113932, + "pass@8": 0.45423242964508603, + "pass@16": 0.5267002530674558, + "pass@32": 0.5911339842264164, + "pass@64": 0.6474984856874543, + "pass@128": 0.6994047619047619 + }, + "20": { + "pass@1": 0.19002619760479042, + "pass@2": 0.2760233715639587, + "pass@4": 0.3645407290031859, + "pass@8": 0.44712179898012794, + "pass@16": 0.5235950630711014, + "pass@32": 0.5922470409190072, + "pass@64": 0.6522222969510175, + "pass@128": 0.7095808383233533 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 586134, + "answer_accuracy": 0.7316139758747404, + "avg_response_len": 302.79132299488737, + "resp_tokens_sum": 242581874, + "pass_at_k": { + "pass@1": 0.7316139758747404, + "pass@2": 0.7816272323759221, + "pass@4": 0.8215453713955919, + "pass@8": 0.8533555308773647, + "pass@16": 0.8788922752379291, + "pass@32": 0.9001364957677545, + "pass@64": 0.9183860626155672, + "pass@128": 0.9335357085796453 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9005597014925373, + "pass@2": 0.931050946057116, + "pass@4": 0.9538955333008745, + "pass@8": 0.9713410311545484, + "pass@16": 0.9821343375480532, + "pass@32": 0.9878797954047336, + "pass@64": 0.9913859266157925, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9932142857142857, + "pass@2": 0.9958703599550055, + "pass@4": 0.9975555758208795, + "pass@8": 0.9988223052875971, + "pass@16": 0.9996540969253741, + "pass@32": 0.9999717161726899, + "pass@64": 0.9999999655157344, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9954901603498543, + "pass@2": 0.998288678519777, + "pass@4": 0.9996947698410874, + "pass@8": 0.9999878398913509, + "pass@16": 0.9999999795348873, + "pass@32": 0.9999999999999929, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998204891329374, + "pass@4": 0.9999901437758876, + "pass@8": 0.9999999784883499, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9567555147058824, + "pass@2": 0.9696738217924962, + "pass@4": 0.9785373726262158, + "pass@8": 0.9853053048265434, + "pass@16": 0.9899959490701858, + "pass@32": 0.9924198270729687, + "pass@64": 0.9941155967279394, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9787066246056783, + "pass@2": 0.9856130772001291, + "pass@4": 0.9893927832167666, + "pass@8": 0.9920141237962015, + "pass@16": 0.993923739223236, + "pass@32": 0.9955704271400222, + "pass@64": 0.9966477762794999, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9710739715189873, + "pass@2": 0.9804854916276287, + "pass@4": 0.9853905750902974, + "pass@8": 0.9884781424038828, + "pass@16": 0.9907223592934364, + "pass@32": 0.993282074816925, + "pass@64": 0.9964584869928289, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9605621246246246, + "pass@2": 0.9780774869357546, + "pass@4": 0.9872390320579295, + "pass@8": 0.9933237632628275, + "pass@16": 0.9967972912739032, + "pass@32": 0.9986820833990249, + "pass@64": 0.9998014084266714, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.95164288585209, + "pass@2": 0.9741879130060515, + "pass@4": 0.984655453940444, + "pass@8": 0.9906493623795749, + "pass@16": 0.9948439227979123, + "pass@32": 0.9975481384153831, + "pass@64": 0.999415604567299, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9049107142857142, + "pass@2": 0.942718097737783, + "pass@4": 0.9644573645556207, + "pass@8": 0.976589220540312, + "pass@16": 0.9832850039268494, + "pass@32": 0.9882524207239414, + "pass@64": 0.9920457050108219, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8531154434250765, + "pass@2": 0.9018159749813379, + "pass@4": 0.9259524213716399, + "pass@8": 0.9384973687647191, + "pass@16": 0.9466330163774461, + "pass@32": 0.9532566422537484, + "pass@64": 0.9597485544795151, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.7125771604938271, + "pass@2": 0.7980232939632549, + "pass@4": 0.8545895863132843, + "pass@8": 0.8871035934478441, + "pass@16": 0.9047211218668199, + "pass@32": 0.9194892020419742, + "pass@64": 0.9369815314607124, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.641853672106825, + "pass@2": 0.736465440197201, + "pass@4": 0.8081795051953817, + "pass@8": 0.8622352195778664, + "pass@16": 0.9002308972209166, + "pass@32": 0.925441310441314, + "pass@64": 0.9411943108318142, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.5632716049382716, + "pass@2": 0.6607102714591229, + "pass@4": 0.7362172228471439, + "pass@8": 0.7940996873385447, + "pass@16": 0.8380536762293106, + "pass@32": 0.8709506567456127, + "pass@64": 0.8953321790010337, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4345271017699115, + "pass@2": 0.5460007142359413, + "pass@4": 0.6376523852660009, + "pass@8": 0.707717594498196, + "pass@16": 0.7617054610490777, + "pass@32": 0.8068025693387675, + "pass@64": 0.8485437906575631, + "pass@128": 0.887905604719764 + }, + "17": { + "pass@1": 0.32876035391566266, + "pass@2": 0.4316730504695946, + "pass@4": 0.5263502491405438, + "pass@8": 0.6092641636632107, + "pass@16": 0.6799035249628819, + "pass@32": 0.7401694244029334, + "pass@64": 0.7903407379642275, + "pass@128": 0.8283132530120482 + }, + "18": { + "pass@1": 0.32202620967741935, + "pass@2": 0.4195766922783844, + "pass@4": 0.5071416249380115, + "pass@8": 0.5826766248109669, + "pass@16": 0.6470575670690685, + "pass@32": 0.6999607829389606, + "pass@64": 0.7450980789610459, + "pass@128": 0.7838709677419354 + }, + "19": { + "pass@1": 0.21190521148036254, + "pass@2": 0.2984186549944095, + "pass@4": 0.3856639179543644, + "pass@8": 0.469373003061341, + "pass@16": 0.5491552351678592, + "pass@32": 0.6242073409545712, + "pass@64": 0.6923793324484625, + "pass@128": 0.7492447129909365 + }, + "20": { + "pass@1": 0.20521302552552553, + "pass@2": 0.2908072935139864, + "pass@4": 0.377205680652531, + "pass@8": 0.4579210041834061, + "pass@16": 0.5327242625772535, + "pass@32": 0.6020922670701319, + "pass@64": 0.6639720954688267, + "pass@128": 0.7147147147147147 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9e1f8bd6c7fd0ba8edf5ef23445ba8d445f31e67 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:256fa0466477900610acd7ff95de85d05ab197279e9de2d2093b65e9de62e023 +size 1792476528 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ab76a93324e4635491f6a6ab62253a500b6df688 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9438151041666667, + "avg_response_len": 201.10614409722223, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9438151041666667, + "pass@2": 0.9672602252843225, + "pass@4": 0.9793120692830055, + "pass@8": 0.9865219893890322, + "pass@16": 0.9911603813413746, + "pass@32": 0.9942249238170957, + "pass@64": 0.9961343348055197, + "pass@128": 0.9972222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8668671875, + "pass@2": 0.9169467273622062, + "pass@4": 0.9463778237720286, + "pass@8": 0.9658439172248066, + "pass@16": 0.9789367184689405, + "pass@32": 0.9875036098314083, + "pass@64": 0.9931097372931847, + "pass@128": 0.997 + }, + "2": { + "pass@1": 0.957046875, + "pass@2": 0.9671719980314959, + "pass@4": 0.9751106509186351, + "pass@8": 0.9804009132053274, + "pass@16": 0.9838877467898254, + "pass@32": 0.9870602824727879, + "pass@64": 0.989887399713069, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.9910078125, + "pass@2": 0.9955991633858269, + "pass@4": 0.9975429010123735, + "pass@8": 0.9986325957282451, + "pass@16": 0.9994065436525951, + "pass@32": 0.9998804533674797, + "pass@64": 0.9999978873639384, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985140625, + "pass@2": 0.990811761811024, + "pass@4": 0.9932380109673792, + "pass@8": 0.9945974608331819, + "pass@16": 0.9956995102839108, + "pass@32": 0.9967100138653072, + "pass@64": 0.9974409053985689, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.949203125, + "pass@2": 0.9661480068897638, + "pass@4": 0.9767052831833517, + "pass@8": 0.9842660545002075, + "pass@16": 0.9897622682141252, + "pass@32": 0.993538156054671, + "pass@64": 0.9960930574294902, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.966203125, + "pass@2": 0.9834298720472446, + "pass@4": 0.9901465293400825, + "pass@8": 0.9934229011843891, + "pass@16": 0.9952443929507612, + "pass@32": 0.9962051689656628, + "pass@64": 0.9967374710294008, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9434375, + "pass@2": 0.9670877214566933, + "pass@4": 0.978003055399325, + "pass@8": 0.9850751598066192, + "pass@16": 0.9899776927511833, + "pass@32": 0.9931160653520409, + "pass@64": 0.9951177111980352, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.9277109375, + "pass@2": 0.9645869832677179, + "pass@4": 0.9818694453505811, + "pass@8": 0.9910390247269298, + "pass@16": 0.9957519803729585, + "pass@32": 0.998255517761501, + "pass@64": 0.9994222120001507, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.90771875, + "pass@2": 0.9535597933070874, + "pass@4": 0.9748149236032984, + "pass@8": 0.9854198772915995, + "pass@16": 0.991776578588067, + "pass@32": 0.9957550466829985, + "pass@64": 0.9974026318238274, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.8668671875, + "2": 0.957046875, + "3": 0.9910078125, + "4": 0.985140625, + "5": 0.949203125, + "6": 0.966203125, + "7": 0.9434375, + "8": 0.9277109375, + "9": 0.90771875 + }, + "per_op_avg_response_len": { + "10": 280.6592734375, + "2": 120.126046875, + "3": 145.57878125, + "4": 159.973328125, + "5": 174.940015625, + "6": 199.8098046875, + "7": 224.402234375, + "8": 244.73475, + "9": 259.7310625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.6592734375, + "2": 120.126046875, + "3": 145.57878125, + "4": 159.973328125, + "5": 174.940015625, + "6": 199.8098046875, + "7": 224.402234375, + "8": 244.73475, + "9": 259.7310625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 357244, + "answer_accuracy": 0.9425764099966227, + "avg_response_len": 191.03214971715636, + "resp_tokens_sum": 72402713, + "pass_at_k": { + "pass@1": 0.9425764099966227, + "pass@2": 0.9658385038971221, + "pass@4": 0.9777205536111451, + "pass@8": 0.9851349531300182, + "pass@16": 0.9899785320660671, + "pass@32": 0.9930368707885944, + "pass@64": 0.9948820560491203, + "pass@128": 0.9959473150962512 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8778576696165191, + "pass@2": 0.9216935376164258, + "pass@4": 0.9474172199713969, + "pass@8": 0.9654188468359517, + "pass@16": 0.9777636401292872, + "pass@32": 0.9863328110936131, + "pass@64": 0.9917817546869228, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9557778426791277, + "pass@2": 0.9654906235435523, + "pass@4": 0.973800176263014, + "pass@8": 0.9798030323439492, + "pass@16": 0.9832815566173658, + "pass@32": 0.9858820824757502, + "pass@64": 0.9882787447648765, + "pass@128": 0.9906542056074766 + }, + "3": { + "pass@1": 0.9922591743119266, + "pass@2": 0.9963278672734716, + "pass@4": 0.9980850739414453, + "pass@8": 0.9992066793724687, + "pass@16": 0.9998328881169319, + "pass@32": 0.9999944749477707, + "pass@64": 0.9999999994985725, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.982568359375, + "pass@2": 0.9903170367864174, + "pass@4": 0.9939585638709225, + "pass@8": 0.9956336116662886, + "pass@16": 0.9965011897205394, + "pass@32": 0.9968440691138198, + "pass@64": 0.9968749622828345, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9425595238095238, + "pass@2": 0.9607877140357454, + "pass@4": 0.9715392290249432, + "pass@8": 0.9791186410445809, + "pass@16": 0.9853992284800553, + "pass@32": 0.9910054778381431, + "pass@64": 0.995247280989616, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9483065119760479, + "pass@2": 0.9705814748455845, + "pass@4": 0.9804762084380167, + "pass@8": 0.9862780220628213, + "pass@16": 0.9896832084934538, + "pass@32": 0.9908509464056335, + "pass@64": 0.9910159624491164, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9368296967455622, + "pass@2": 0.9643732819270364, + "pass@4": 0.9772801360258956, + "pass@8": 0.9849902895915007, + "pass@16": 0.9902276572440275, + "pass@32": 0.9929242140601698, + "pass@64": 0.9939053564147182, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.9312650240384616, + "pass@2": 0.9645184263325259, + "pass@4": 0.9803785148851585, + "pass@8": 0.9896741609551327, + "pass@16": 0.9946263420996423, + "pass@32": 0.9971043502224575, + "pass@64": 0.9983853532572591, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9206646126760564, + "pass@2": 0.9607331984030167, + "pass@4": 0.9777831326541929, + "pass@8": 0.986769595696414, + "pass@16": 0.9927609271921384, + "pass@32": 0.9964592233943435, + "pass@64": 0.9984671060065928, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 369209, + "answer_accuracy": 0.9451000368610747, + "avg_response_len": 199.15424311926606, + "resp_tokens_sum": 77800800, + "pass_at_k": { + "pass@1": 0.9451000368610747, + "pass@2": 0.9688266326585931, + "pass@4": 0.9802850637997751, + "pass@8": 0.9867527412941159, + "pass@16": 0.9909703255506649, + "pass@32": 0.9938592179327799, + "pass@64": 0.9956330191135236, + "pass@128": 0.9967234600262124 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8696798312883436, + "pass@2": 0.9186146653543302, + "pass@4": 0.9464048984866151, + "pass@8": 0.9644575325567745, + "pass@16": 0.9772704068767525, + "pass@32": 0.9862547560604297, + "pass@64": 0.9922762044109446, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9586103723404256, + "pass@2": 0.9672702564440083, + "pass@4": 0.9736437885423895, + "pass@8": 0.9779361963612356, + "pass@16": 0.9810863329863315, + "pass@32": 0.9836883585763785, + "pass@64": 0.9859087861062281, + "pass@128": 0.9878419452887538 + }, + "3": { + "pass@1": 0.9908617424242424, + "pass@2": 0.9960514346218085, + "pass@4": 0.9976872544341046, + "pass@8": 0.9984464684143097, + "pass@16": 0.9992439929783652, + "pass@32": 0.9998493181625818, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9850684171597633, + "pass@2": 0.9900158848949354, + "pass@4": 0.9916054067575871, + "pass@8": 0.9926058357250064, + "pass@16": 0.9935995492288452, + "pass@32": 0.994621258385564, + "pass@64": 0.9955578558086908, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.9645833333333333, + "pass@2": 0.9754475493552436, + "pass@4": 0.982398531976981, + "pass@8": 0.9883361791394907, + "pass@16": 0.9929284629109592, + "pass@32": 0.9956182396383584, + "pass@64": 0.9968886353429263, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9753313037249284, + "pass@2": 0.9916807441960164, + "pass@4": 0.9966340057242126, + "pass@8": 0.9980058334175127, + "pass@16": 0.9987355344474242, + "pass@32": 0.9995102342609697, + "pass@64": 0.9999603905476593, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9460350433526011, + "pass@2": 0.9703241631013609, + "pass@4": 0.9808562748298078, + "pass@8": 0.9870636604011149, + "pass@16": 0.9908352195978022, + "pass@32": 0.9934234406974602, + "pass@64": 0.9956567243821888, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9247579225352113, + "pass@2": 0.9658260092048359, + "pass@4": 0.9840326755810451, + "pass@8": 0.9923905005669404, + "pass@16": 0.9966280877479383, + "pass@32": 0.9989143053771641, + "pass@64": 0.9998999175263501, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8886835703592815, + "pass@2": 0.9419457099580372, + "pass@4": 0.9674046631147154, + "pass@8": 0.9800132434000822, + "pass@16": 0.9872327356476097, + "pass@32": 0.9919673623743022, + "pass@64": 0.9938730559494572, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 360822, + "answer_accuracy": 0.9437301221961835, + "avg_response_len": 213.08682677017075, + "resp_tokens_sum": 81470765, + "pass_at_k": { + "pass@1": 0.9437301221961835, + "pass@2": 0.9670690775512808, + "pass@4": 0.9798955639730691, + "pass@8": 0.9876611790604866, + "pass@16": 0.9925261349327492, + "pass@32": 0.9957762995708028, + "pass@64": 0.9978879380494621, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.853008395522388, + "pass@2": 0.9105201110588794, + "pass@4": 0.9452996695375757, + "pass@8": 0.9676232015937645, + "pass@16": 0.9817453487262422, + "pass@32": 0.9899036907312646, + "pass@64": 0.9952647158697007, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9567410714285715, + "pass@2": 0.9686216957255346, + "pass@4": 0.9776913940221758, + "pass@8": 0.9832660892002095, + "pass@16": 0.987077038751852, + "pass@32": 0.9913104686469809, + "pass@64": 0.9951026628988415, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9899553571428571, + "pass@2": 0.9944693217556989, + "pass@4": 0.9968871366480065, + "pass@8": 0.9982643644219408, + "pass@16": 0.9991564768381861, + "pass@32": 0.9998017057315063, + "pass@64": 0.9999957879397594, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9876187865497076, + "pass@2": 0.992061230832988, + "pass@4": 0.9941773188439165, + "pass@8": 0.995596294224904, + "pass@16": 0.9970248009765742, + "pass@32": 0.9986489076449245, + "pass@64": 0.9998314684348667, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9397518382352941, + "pass@2": 0.9616778601204263, + "pass@4": 0.9757145073777543, + "pass@8": 0.9849050022589428, + "pass@16": 0.9905917398194917, + "pass@32": 0.9937739348835954, + "pass@64": 0.9960693668366222, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9750098580441641, + "pass@2": 0.9878835322536579, + "pass@4": 0.9931930843029477, + "pass@8": 0.9959053815542414, + "pass@16": 0.9972602517091385, + "pass@32": 0.9982078268425956, + "pass@64": 0.9992175497484641, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9476611946202531, + "pass@2": 0.9664474421907704, + "pass@4": 0.9756522099096789, + "pass@8": 0.9829886564110945, + "pass@16": 0.9887713880438683, + "pass@32": 0.9929847168303246, + "pass@64": 0.9958242851064026, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9275290915915916, + "pass@2": 0.9633303332860415, + "pass@4": 0.9809602068322539, + "pass@8": 0.990877053476471, + "pass@16": 0.9958726441061623, + "pass@32": 0.998631778027631, + "pass@64": 0.9998844176036991, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9133842443729904, + "pass@2": 0.9578445040129633, + "pass@4": 0.9793850611921092, + "pass@8": 0.9896856833560922, + "pass@16": 0.99553284800179, + "pass@32": 0.9990190461253682, + "pass@64": 0.9999781623934679, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4157703125, + "avg_response_len": 354.7530515625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4157703125, + "pass@2": 0.5163415477362168, + "pass@4": 0.6016587555118015, + "pass@8": 0.6705138154844426, + "pass@16": 0.7255644881819717, + "pass@32": 0.7701611371083099, + "pass@64": 0.8075568916472131, + "pass@128": 0.84 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8145625, + "pass@2": 0.8910001230314964, + "pass@4": 0.9339941961942249, + "pass@8": 0.9562498410099743, + "pass@16": 0.968104646908941, + "pass@32": 0.9759598827930538, + "pass@64": 0.9821180515686281, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.6921640625, + "pass@2": 0.7999575541338584, + "pass@4": 0.8684834830333712, + "pass@8": 0.9089916512366105, + "pass@16": 0.9343303869507086, + "pass@32": 0.9517072480478936, + "pass@64": 0.9645990784975665, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.5129375, + "pass@2": 0.6359467273622038, + "pass@4": 0.7317865559617536, + "pass@8": 0.800907676240289, + "pass@16": 0.8512054367102967, + "pass@32": 0.8888475514215549, + "pass@64": 0.9179486427319447, + "pass@128": 0.942 + }, + "14": { + "pass@1": 0.4916484375, + "pass@2": 0.6030867372047239, + "pass@4": 0.6943520789276341, + "pass@8": 0.7669748469059653, + "pass@16": 0.8222851322470838, + "pass@32": 0.8626863906492007, + "pass@64": 0.8936645234495811, + "pass@128": 0.921 + }, + "15": { + "pass@1": 0.4215546875, + "pass@2": 0.5421889763779524, + "pass@4": 0.6432598669853773, + "pass@8": 0.7223823027021813, + "pass@16": 0.7820814844216731, + "pass@32": 0.8263602444987515, + "pass@64": 0.8602773565402181, + "pass@128": 0.889 + }, + "16": { + "pass@1": 0.3003203125, + "pass@2": 0.406388656496063, + "pass@4": 0.5046786018935129, + "pass@8": 0.5872669015572977, + "pass@16": 0.6546303956904447, + "pass@32": 0.7114309394558879, + "pass@64": 0.7612586298598292, + "pass@128": 0.808 + }, + "17": { + "pass@1": 0.2711796875, + "pass@2": 0.3733278789370078, + "pass@4": 0.46968226387326545, + "pass@8": 0.5526845580293714, + "pass@16": 0.6221160315050789, + "pass@32": 0.6815997112308478, + "pass@64": 0.7344530263245084, + "pass@128": 0.781 + }, + "18": { + "pass@1": 0.2673671875, + "pass@2": 0.365512795275591, + "pass@4": 0.46002104846269143, + "pass@8": 0.5430447218051798, + "pass@16": 0.6105401606357511, + "pass@32": 0.6632556246757224, + "pass@64": 0.7066100861091823, + "pass@128": 0.745 + }, + "19": { + "pass@1": 0.1928671875, + "pass@2": 0.2710813238188976, + "pass@4": 0.35185807358455123, + "pass@8": 0.4293775982246976, + "pass@16": 0.500870627555131, + "pass@32": 0.565888710514292, + "pass@64": 0.6233455846948991, + "pass@128": 0.672 + }, + "20": { + "pass@1": 0.1931015625, + "pass@2": 0.27492470472440933, + "pass@4": 0.3584713862017249, + "pass@8": 0.43725805713284427, + "pass@16": 0.5094805791944168, + "pass@32": 0.5738750677958954, + "pass@64": 0.6312939366956709, + "pass@128": 0.681 + } + }, + "per_op_accuracy": { + "11": 0.8145625, + "12": 0.6921640625, + "13": 0.5129375, + "14": 0.4916484375, + "15": 0.4215546875, + "16": 0.3003203125, + "17": 0.2711796875, + "18": 0.2673671875, + "19": 0.1928671875, + "20": 0.1931015625 + }, + "per_op_avg_response_len": { + "11": 297.857515625, + "12": 305.12346875, + "13": 319.913203125, + "14": 344.86790625, + "15": 360.1027734375, + "16": 362.2561640625, + "17": 379.60903125, + "18": 388.6507265625, + "19": 387.6355703125, + "20": 401.51415625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.857515625, + "12": 305.12346875, + "13": 319.913203125, + "14": 344.86790625, + "15": 360.1027734375, + "16": 362.2561640625, + "17": 379.60903125, + "18": 388.6507265625, + "19": 387.6355703125, + "20": 401.51415625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 183558, + "answer_accuracy": 0.42616549034175333, + "avg_response_len": 336.0189543090639, + "resp_tokens_sum": 144730084, + "pass_at_k": { + "pass@1": 0.42616549034175333, + "pass@2": 0.5246652738940701, + "pass@4": 0.6056691621663947, + "pass@8": 0.6702889406318995, + "pass@16": 0.7219717847837656, + "pass@32": 0.7632459440515996, + "pass@64": 0.7981730421110546, + "pass@128": 0.8291233283803864 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8213748229461756, + "pass@2": 0.8935474336954341, + "pass@4": 0.9340763818297493, + "pass@8": 0.9559787207718502, + "pass@16": 0.9685268552496731, + "pass@32": 0.9767709530477564, + "pass@64": 0.982877126664917, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6984463276836158, + "pass@2": 0.8106034382089947, + "pass@4": 0.8795497037976184, + "pass@8": 0.9196428046531496, + "pass@16": 0.9460275011724377, + "pass@32": 0.9631035966250991, + "pass@64": 0.9734107084727417, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.5377644984326019, + "pass@2": 0.6605780860464542, + "pass@4": 0.7524930616039762, + "pass@8": 0.8165367240179848, + "pass@16": 0.8641324613574773, + "pass@32": 0.9023888533468492, + "pass@64": 0.9334225133172553, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.5067659657320872, + "pass@2": 0.6147987342703658, + "pass@4": 0.6982890669624235, + "pass@8": 0.7658124058064986, + "pass@16": 0.8180146824253777, + "pass@32": 0.8548311069597581, + "pass@64": 0.8841931928289547, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.4333227962427746, + "pass@2": 0.5478261828091572, + "pass@4": 0.6419825939387632, + "pass@8": 0.7153113400084069, + "pass@16": 0.7722291645122227, + "pass@32": 0.8171855305285305, + "pass@64": 0.854938200858591, + "pass@128": 0.8872832369942196 + }, + "16": { + "pass@1": 0.3077620967741935, + "pass@2": 0.4179562484124969, + "pass@4": 0.5183253746507492, + "pass@8": 0.5993537141790105, + "pass@16": 0.6641207838509781, + "pass@32": 0.7170970581981619, + "pass@64": 0.7597654177085406, + "pass@128": 0.7935483870967742 + }, + "17": { + "pass@1": 0.2698022037572254, + "pass@2": 0.36404415195029793, + "pass@4": 0.45026019336094525, + "pass@8": 0.5247725639243038, + "pass@16": 0.5872757336442391, + "pass@32": 0.6408713150622185, + "pass@64": 0.6923314852463324, + "pass@128": 0.7427745664739884 + }, + "18": { + "pass@1": 0.2632589285714286, + "pass@2": 0.3572957677165353, + "pass@4": 0.4467472826610956, + "pass@8": 0.5262610157224527, + "pass@16": 0.589722143816733, + "pass@32": 0.6356788942331097, + "pass@64": 0.6732118806686305, + "pass@128": 0.7085714285714285 + }, + "19": { + "pass@1": 0.19390484234234234, + "pass@2": 0.2684835425977157, + "pass@4": 0.34286924128853263, + "pass@8": 0.4151729621852111, + "pass@16": 0.48393435094925585, + "pass@32": 0.5479942111320676, + "pass@64": 0.6062296128855615, + "pass@128": 0.6606606606606606 + }, + "20": { + "pass@1": 0.20190503003003002, + "pass@2": 0.286834447636613, + "pass@4": 0.37134299935481024, + "pass@8": 0.4471309689317958, + "pass@16": 0.5122863221912953, + "pass@32": 0.5662567887278481, + "pass@64": 0.6136344244399008, + "pass@128": 0.6576576576576577 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 175159, + "answer_accuracy": 0.40690743012191494, + "avg_response_len": 350.1974961901576, + "resp_tokens_sum": 150747415, + "pass_at_k": { + "pass@1": 0.40690743012191494, + "pass@2": 0.5087476381464827, + "pass@4": 0.5967527069263558, + "pass@8": 0.667915576478635, + "pass@16": 0.7245869907436522, + "pass@32": 0.7707101532352331, + "pass@64": 0.8086176122408342, + "pass@128": 0.8412132024977699 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8057934864457831, + "pass@2": 0.8846412964851539, + "pass@4": 0.929690460096855, + "pass@8": 0.9521490248359574, + "pass@16": 0.9636872668525197, + "pass@32": 0.9720114728755084, + "pass@64": 0.9792833961748288, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.6970023510971787, + "pass@2": 0.7992438383975512, + "pass@4": 0.8650445735819062, + "pass@8": 0.9031356523629713, + "pass@16": 0.9249747945390557, + "pass@32": 0.9407246495295145, + "pass@64": 0.9545253669368997, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.48713235294117646, + "pass@2": 0.6071869692758992, + "pass@4": 0.7042174234523209, + "pass@8": 0.775814783223066, + "pass@16": 0.8292051212942726, + "pass@32": 0.8707959721079798, + "pass@64": 0.9030591892971792, + "pass@128": 0.9299719887955182 + }, + "14": { + "pass@1": 0.48384959795321636, + "pass@2": 0.5952749430169916, + "pass@4": 0.6884485705734146, + "pass@8": 0.7624990556025817, + "pass@16": 0.8195176330850231, + "pass@32": 0.86363662023738, + "pass@64": 0.8976129475131868, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.3916903409090909, + "pass@2": 0.5208993229539489, + "pass@4": 0.6331746480553567, + "pass@8": 0.7205971077124683, + "pass@16": 0.7829411999553293, + "pass@32": 0.8238903194155507, + "pass@64": 0.8522462435594557, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.30088141025641024, + "pass@2": 0.4054595839782847, + "pass@4": 0.500781167311351, + "pass@8": 0.5809003310690419, + "pass@16": 0.6454340647690915, + "pass@32": 0.6993132560546717, + "pass@64": 0.7482662812431827, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.27986704192546585, + "pass@2": 0.3884398536460114, + "pass@4": 0.49235009938586827, + "pass@8": 0.583732081509763, + "pass@16": 0.6622393198782099, + "pass@32": 0.7289934318444642, + "pass@64": 0.7808316991010685, + "pass@128": 0.8167701863354038 + }, + "18": { + "pass@1": 0.26505055147058826, + "pass@2": 0.37142195460861493, + "pass@4": 0.47396713728798606, + "pass@8": 0.5602206902178573, + "pass@16": 0.6280354105963233, + "pass@32": 0.6817861507678503, + "pass@64": 0.7276384154090412, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.19022042410714285, + "pass@2": 0.26708416818991365, + "pass@4": 0.34927864904163775, + "pass@8": 0.4289785213982712, + "pass@16": 0.501354952736681, + "pass@32": 0.565596716464881, + "pass@64": 0.619600156910572, + "pass@128": 0.6607142857142857 + }, + "20": { + "pass@1": 0.18013192365269462, + "pass@2": 0.2592645663397615, + "pass@4": 0.34027802999176016, + "pass@8": 0.4190993109617173, + "pass@16": 0.4949920405942872, + "pass@32": 0.5656735198870328, + "pass@64": 0.627234567640334, + "pass@128": 0.6796407185628742 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 173469, + "answer_accuracy": 0.4141890472188264, + "avg_response_len": 378.7018810169621, + "resp_tokens_sum": 158606407, + "pass_at_k": { + "pass@1": 0.4141890472188264, + "pass@2": 0.5155863458502609, + "pass@4": 0.602576855450756, + "pass@8": 0.6734155824940252, + "pass@16": 0.7302639902662885, + "pass@32": 0.7767085953604392, + "pass@64": 0.8161172371034107, + "pass@128": 0.8499388753056235 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8161706349206349, + "pass@2": 0.8948475190601176, + "pass@4": 0.9384380973211678, + "pass@8": 0.9608757978792809, + "pass@16": 0.9722872838437171, + "pass@32": 0.9792124678492932, + "pass@64": 0.9842550423043461, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.6806431574923547, + "pass@2": 0.7891289069084256, + "pass@4": 0.8598583147290075, + "pass@8": 0.9031737776318244, + "pass@16": 0.9307941653752537, + "pass@32": 0.9500838276534974, + "pass@64": 0.9648871426463134, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5169270833333334, + "pass@2": 0.6433844749441041, + "pass@4": 0.7417766948691599, + "pass@8": 0.8131684681108555, + "pass@16": 0.8627189914666871, + "pass@32": 0.895405386300501, + "pass@64": 0.9191195382859483, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.48516320474777447, + "pass@2": 0.5998584955255963, + "pass@4": 0.6965931077050082, + "pass@8": 0.7726242956261022, + "pass@16": 0.8291613907224347, + "pass@32": 0.8692043952342279, + "pass@64": 0.8986791706290098, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.43940489969135804, + "pass@2": 0.5578529029357437, + "pass@4": 0.6548958506922737, + "pass@8": 0.7317516466486336, + "pass@16": 0.7917271528246511, + "pass@32": 0.8386735973109505, + "pass@64": 0.8741588847793955, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.2929341814159292, + "pass@2": 0.3967725826307111, + "pass@4": 0.4962346490427633, + "pass@8": 0.5828059998718874, + "pass@16": 0.6554737344032179, + "pass@32": 0.7187961608827936, + "pass@64": 0.7760763588608376, + "pass@128": 0.8289085545722714 + }, + "17": { + "pass@1": 0.26418957078313254, + "pass@2": 0.36834629363912336, + "pass@4": 0.46793826797629223, + "pass@8": 0.551661206823551, + "pass@16": 0.6195107429620204, + "pass@32": 0.678079370980133, + "pass@64": 0.733369052164855, + "pass@128": 0.786144578313253 + }, + "18": { + "pass@1": 0.2745463709677419, + "pass@2": 0.36830907099314175, + "pass@4": 0.4597118479141718, + "pass@8": 0.5431559084782237, + "pass@16": 0.6148560345069214, + "pass@32": 0.6740668401066608, + "pass@64": 0.7212544085035065, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.1945100075528701, + "pass@2": 0.2777523372267288, + "pass@4": 0.36351960772652664, + "pass@8": 0.4440731680580169, + "pass@16": 0.5174175968867194, + "pass@32": 0.5841877385955095, + "pass@64": 0.6443669815167831, + "pass@128": 0.6948640483383686 + }, + "20": { + "pass@1": 0.1973066816816817, + "pass@2": 0.27872212763945037, + "pass@4": 0.363847763961937, + "pass@8": 0.4455984222743029, + "pass@16": 0.5212068839225962, + "pass@32": 0.5897195240458061, + "pass@64": 0.6530250083042405, + "pass@128": 0.7057057057057057 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6658967927631579, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6658967927631579, + "pass@2": 0.7299346055221402, + "pass@4": 0.7805471672981394, + "pass@8": 0.8202018978603093, + "pass@16": 0.8513730691522174, + "pass@32": 0.8762966150230169, + "pass@64": 0.896883048932726, + "pass@128": 0.9144736842105263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8668671875, + "pass@2": 0.9169467273622062, + "pass@4": 0.9463778237720286, + "pass@8": 0.9658439172248066, + "pass@16": 0.9789367184689405, + "pass@32": 0.9875036098314083, + "pass@64": 0.9931097372931847, + "pass@128": 0.997 + }, + "2": { + "pass@1": 0.957046875, + "pass@2": 0.9671719980314959, + "pass@4": 0.9751106509186351, + "pass@8": 0.9804009132053274, + "pass@16": 0.9838877467898254, + "pass@32": 0.9870602824727879, + "pass@64": 0.989887399713069, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.9910078125, + "pass@2": 0.9955991633858269, + "pass@4": 0.9975429010123735, + "pass@8": 0.9986325957282451, + "pass@16": 0.9994065436525951, + "pass@32": 0.9998804533674797, + "pass@64": 0.9999978873639384, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985140625, + "pass@2": 0.990811761811024, + "pass@4": 0.9932380109673792, + "pass@8": 0.9945974608331819, + "pass@16": 0.9956995102839108, + "pass@32": 0.9967100138653072, + "pass@64": 0.9974409053985689, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.949203125, + "pass@2": 0.9661480068897638, + "pass@4": 0.9767052831833517, + "pass@8": 0.9842660545002075, + "pass@16": 0.9897622682141252, + "pass@32": 0.993538156054671, + "pass@64": 0.9960930574294902, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.966203125, + "pass@2": 0.9834298720472446, + "pass@4": 0.9901465293400825, + "pass@8": 0.9934229011843891, + "pass@16": 0.9952443929507612, + "pass@32": 0.9962051689656628, + "pass@64": 0.9967374710294008, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9434375, + "pass@2": 0.9670877214566933, + "pass@4": 0.978003055399325, + "pass@8": 0.9850751598066192, + "pass@16": 0.9899776927511833, + "pass@32": 0.9931160653520409, + "pass@64": 0.9951177111980352, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.9277109375, + "pass@2": 0.9645869832677179, + "pass@4": 0.9818694453505811, + "pass@8": 0.9910390247269298, + "pass@16": 0.9957519803729585, + "pass@32": 0.998255517761501, + "pass@64": 0.9994222120001507, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.90771875, + "pass@2": 0.9535597933070874, + "pass@4": 0.9748149236032984, + "pass@8": 0.9854198772915995, + "pass@16": 0.991776578588067, + "pass@32": 0.9957550466829985, + "pass@64": 0.9974026318238274, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.8145625, + "pass@2": 0.8910001230314964, + "pass@4": 0.9339941961942249, + "pass@8": 0.9562498410099743, + "pass@16": 0.968104646908941, + "pass@32": 0.9759598827930538, + "pass@64": 0.9821180515686281, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.6921640625, + "pass@2": 0.7999575541338584, + "pass@4": 0.8684834830333712, + "pass@8": 0.9089916512366105, + "pass@16": 0.9343303869507086, + "pass@32": 0.9517072480478936, + "pass@64": 0.9645990784975665, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.5129375, + "pass@2": 0.6359467273622038, + "pass@4": 0.7317865559617536, + "pass@8": 0.800907676240289, + "pass@16": 0.8512054367102967, + "pass@32": 0.8888475514215549, + "pass@64": 0.9179486427319447, + "pass@128": 0.942 + }, + "14": { + "pass@1": 0.4916484375, + "pass@2": 0.6030867372047239, + "pass@4": 0.6943520789276341, + "pass@8": 0.7669748469059653, + "pass@16": 0.8222851322470838, + "pass@32": 0.8626863906492007, + "pass@64": 0.8936645234495811, + "pass@128": 0.921 + }, + "15": { + "pass@1": 0.4215546875, + "pass@2": 0.5421889763779524, + "pass@4": 0.6432598669853773, + "pass@8": 0.7223823027021813, + "pass@16": 0.7820814844216731, + "pass@32": 0.8263602444987515, + "pass@64": 0.8602773565402181, + "pass@128": 0.889 + }, + "16": { + "pass@1": 0.3003203125, + "pass@2": 0.406388656496063, + "pass@4": 0.5046786018935129, + "pass@8": 0.5872669015572977, + "pass@16": 0.6546303956904447, + "pass@32": 0.7114309394558879, + "pass@64": 0.7612586298598292, + "pass@128": 0.808 + }, + "17": { + "pass@1": 0.2711796875, + "pass@2": 0.3733278789370078, + "pass@4": 0.46968226387326545, + "pass@8": 0.5526845580293714, + "pass@16": 0.6221160315050789, + "pass@32": 0.6815997112308478, + "pass@64": 0.7344530263245084, + "pass@128": 0.781 + }, + "18": { + "pass@1": 0.2673671875, + "pass@2": 0.365512795275591, + "pass@4": 0.46002104846269143, + "pass@8": 0.5430447218051798, + "pass@16": 0.6105401606357511, + "pass@32": 0.6632556246757224, + "pass@64": 0.7066100861091823, + "pass@128": 0.745 + }, + "19": { + "pass@1": 0.1928671875, + "pass@2": 0.2710813238188976, + "pass@4": 0.35185807358455123, + "pass@8": 0.4293775982246976, + "pass@16": 0.500870627555131, + "pass@32": 0.565888710514292, + "pass@64": 0.6233455846948991, + "pass@128": 0.672 + }, + "20": { + "pass@1": 0.1931015625, + "pass@2": 0.27492470472440933, + "pass@4": 0.3584713862017249, + "pass@8": 0.43725805713284427, + "pass@16": 0.5094805791944168, + "pass@32": 0.5738750677958954, + "pass@64": 0.6312939366956709, + "pass@128": 0.681 + } + }, + "per_op_accuracy": { + "10": 0.8668671875, + "2": 0.957046875, + "3": 0.9910078125, + "4": 0.985140625, + "5": 0.949203125, + "6": 0.966203125, + "7": 0.9434375, + "8": 0.9277109375, + "9": 0.90771875, + "11": 0.8145625, + "12": 0.6921640625, + "13": 0.5129375, + "14": 0.4916484375, + "15": 0.4215546875, + "16": 0.3003203125, + "17": 0.2711796875, + "18": 0.2673671875, + "19": 0.1928671875, + "20": 0.1931015625 + }, + "per_op_avg_response_len": { + "10": 280.6592734375, + "2": 120.126046875, + "3": 145.57878125, + "4": 159.973328125, + "5": 174.940015625, + "6": 199.8098046875, + "7": 224.402234375, + "8": 244.73475, + "9": 259.7310625, + "11": 297.857515625, + "12": 305.12346875, + "13": 319.913203125, + "14": 344.86790625, + "15": 360.1027734375, + "16": 362.2561640625, + "17": 379.60903125, + "18": 388.6507265625, + "19": 387.6355703125, + "20": 401.51415625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.6592734375, + "2": 120.126046875, + "3": 145.57878125, + "4": 159.973328125, + "5": 174.940015625, + "6": 199.8098046875, + "7": 224.402234375, + "8": 244.73475, + "9": 259.7310625, + "11": 297.857515625, + "12": 305.12346875, + "13": 319.913203125, + "14": 344.86790625, + "15": 360.1027734375, + "16": 362.2561640625, + "17": 379.60903125, + "18": 388.6507265625, + "19": 387.6355703125, + "20": 401.51415625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 540802, + "answer_accuracy": 0.6678810662345874, + "avg_response_len": 268.15522866938034, + "resp_tokens_sum": 217132797, + "pass_at_k": { + "pass@1": 0.6678810662345874, + "pass@2": 0.731164473078238, + "pass@4": 0.779814620602667, + "pass@8": 0.8176583751887917, + "pass@16": 0.8474172445850535, + "pass@32": 0.870803790094637, + "pass@64": 0.8902462938136585, + "pass@128": 0.9072083465064812 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8778576696165191, + "pass@2": 0.9216935376164258, + "pass@4": 0.9474172199713969, + "pass@8": 0.9654188468359517, + "pass@16": 0.9777636401292872, + "pass@32": 0.9863328110936131, + "pass@64": 0.9917817546869228, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9557778426791277, + "pass@2": 0.9654906235435523, + "pass@4": 0.973800176263014, + "pass@8": 0.9798030323439492, + "pass@16": 0.9832815566173658, + "pass@32": 0.9858820824757502, + "pass@64": 0.9882787447648765, + "pass@128": 0.9906542056074766 + }, + "3": { + "pass@1": 0.9922591743119266, + "pass@2": 0.9963278672734716, + "pass@4": 0.9980850739414453, + "pass@8": 0.9992066793724687, + "pass@16": 0.9998328881169319, + "pass@32": 0.9999944749477707, + "pass@64": 0.9999999994985725, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.982568359375, + "pass@2": 0.9903170367864174, + "pass@4": 0.9939585638709225, + "pass@8": 0.9956336116662886, + "pass@16": 0.9965011897205394, + "pass@32": 0.9968440691138198, + "pass@64": 0.9968749622828345, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9425595238095238, + "pass@2": 0.9607877140357454, + "pass@4": 0.9715392290249432, + "pass@8": 0.9791186410445809, + "pass@16": 0.9853992284800553, + "pass@32": 0.9910054778381431, + "pass@64": 0.995247280989616, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9483065119760479, + "pass@2": 0.9705814748455845, + "pass@4": 0.9804762084380167, + "pass@8": 0.9862780220628213, + "pass@16": 0.9896832084934538, + "pass@32": 0.9908509464056335, + "pass@64": 0.9910159624491164, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9368296967455622, + "pass@2": 0.9643732819270364, + "pass@4": 0.9772801360258956, + "pass@8": 0.9849902895915007, + "pass@16": 0.9902276572440275, + "pass@32": 0.9929242140601698, + "pass@64": 0.9939053564147182, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.9312650240384616, + "pass@2": 0.9645184263325259, + "pass@4": 0.9803785148851585, + "pass@8": 0.9896741609551327, + "pass@16": 0.9946263420996423, + "pass@32": 0.9971043502224575, + "pass@64": 0.9983853532572591, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9206646126760564, + "pass@2": 0.9607331984030167, + "pass@4": 0.9777831326541929, + "pass@8": 0.986769595696414, + "pass@16": 0.9927609271921384, + "pass@32": 0.9964592233943435, + "pass@64": 0.9984671060065928, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8213748229461756, + "pass@2": 0.8935474336954341, + "pass@4": 0.9340763818297493, + "pass@8": 0.9559787207718502, + "pass@16": 0.9685268552496731, + "pass@32": 0.9767709530477564, + "pass@64": 0.982877126664917, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6984463276836158, + "pass@2": 0.8106034382089947, + "pass@4": 0.8795497037976184, + "pass@8": 0.9196428046531496, + "pass@16": 0.9460275011724377, + "pass@32": 0.9631035966250991, + "pass@64": 0.9734107084727417, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.5377644984326019, + "pass@2": 0.6605780860464542, + "pass@4": 0.7524930616039762, + "pass@8": 0.8165367240179848, + "pass@16": 0.8641324613574773, + "pass@32": 0.9023888533468492, + "pass@64": 0.9334225133172553, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.5067659657320872, + "pass@2": 0.6147987342703658, + "pass@4": 0.6982890669624235, + "pass@8": 0.7658124058064986, + "pass@16": 0.8180146824253777, + "pass@32": 0.8548311069597581, + "pass@64": 0.8841931928289547, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.4333227962427746, + "pass@2": 0.5478261828091572, + "pass@4": 0.6419825939387632, + "pass@8": 0.7153113400084069, + "pass@16": 0.7722291645122227, + "pass@32": 0.8171855305285305, + "pass@64": 0.854938200858591, + "pass@128": 0.8872832369942196 + }, + "16": { + "pass@1": 0.3077620967741935, + "pass@2": 0.4179562484124969, + "pass@4": 0.5183253746507492, + "pass@8": 0.5993537141790105, + "pass@16": 0.6641207838509781, + "pass@32": 0.7170970581981619, + "pass@64": 0.7597654177085406, + "pass@128": 0.7935483870967742 + }, + "17": { + "pass@1": 0.2698022037572254, + "pass@2": 0.36404415195029793, + "pass@4": 0.45026019336094525, + "pass@8": 0.5247725639243038, + "pass@16": 0.5872757336442391, + "pass@32": 0.6408713150622185, + "pass@64": 0.6923314852463324, + "pass@128": 0.7427745664739884 + }, + "18": { + "pass@1": 0.2632589285714286, + "pass@2": 0.3572957677165353, + "pass@4": 0.4467472826610956, + "pass@8": 0.5262610157224527, + "pass@16": 0.589722143816733, + "pass@32": 0.6356788942331097, + "pass@64": 0.6732118806686305, + "pass@128": 0.7085714285714285 + }, + "19": { + "pass@1": 0.19390484234234234, + "pass@2": 0.2684835425977157, + "pass@4": 0.34286924128853263, + "pass@8": 0.4151729621852111, + "pass@16": 0.48393435094925585, + "pass@32": 0.5479942111320676, + "pass@64": 0.6062296128855615, + "pass@128": 0.6606606606606606 + }, + "20": { + "pass@1": 0.20190503003003002, + "pass@2": 0.286834447636613, + "pass@4": 0.37134299935481024, + "pass@8": 0.4471309689317958, + "pass@16": 0.5122863221912953, + "pass@32": 0.5662567887278481, + "pass@64": 0.6136344244399008, + "pass@128": 0.6576576576576577 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 544368, + "answer_accuracy": 0.6629579111457521, + "avg_response_len": 278.3371675272798, + "resp_tokens_sum": 228548215, + "pass_at_k": { + "pass@1": 0.6629579111457521, + "pass@2": 0.7276347918878587, + "pass@4": 0.7792220371177242, + "pass@8": 0.8196055261305205, + "pass@16": 0.8513215095014183, + "pass@32": 0.8768755383415355, + "pass@64": 0.8975920505534613, + "pass@128": 0.9151987529228371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8696798312883436, + "pass@2": 0.9186146653543302, + "pass@4": 0.9464048984866151, + "pass@8": 0.9644575325567745, + "pass@16": 0.9772704068767525, + "pass@32": 0.9862547560604297, + "pass@64": 0.9922762044109446, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9586103723404256, + "pass@2": 0.9672702564440083, + "pass@4": 0.9736437885423895, + "pass@8": 0.9779361963612356, + "pass@16": 0.9810863329863315, + "pass@32": 0.9836883585763785, + "pass@64": 0.9859087861062281, + "pass@128": 0.9878419452887538 + }, + "3": { + "pass@1": 0.9908617424242424, + "pass@2": 0.9960514346218085, + "pass@4": 0.9976872544341046, + "pass@8": 0.9984464684143097, + "pass@16": 0.9992439929783652, + "pass@32": 0.9998493181625818, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9850684171597633, + "pass@2": 0.9900158848949354, + "pass@4": 0.9916054067575871, + "pass@8": 0.9926058357250064, + "pass@16": 0.9935995492288452, + "pass@32": 0.994621258385564, + "pass@64": 0.9955578558086908, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.9645833333333333, + "pass@2": 0.9754475493552436, + "pass@4": 0.982398531976981, + "pass@8": 0.9883361791394907, + "pass@16": 0.9929284629109592, + "pass@32": 0.9956182396383584, + "pass@64": 0.9968886353429263, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9753313037249284, + "pass@2": 0.9916807441960164, + "pass@4": 0.9966340057242126, + "pass@8": 0.9980058334175127, + "pass@16": 0.9987355344474242, + "pass@32": 0.9995102342609697, + "pass@64": 0.9999603905476593, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9460350433526011, + "pass@2": 0.9703241631013609, + "pass@4": 0.9808562748298078, + "pass@8": 0.9870636604011149, + "pass@16": 0.9908352195978022, + "pass@32": 0.9934234406974602, + "pass@64": 0.9956567243821888, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9247579225352113, + "pass@2": 0.9658260092048359, + "pass@4": 0.9840326755810451, + "pass@8": 0.9923905005669404, + "pass@16": 0.9966280877479383, + "pass@32": 0.9989143053771641, + "pass@64": 0.9998999175263501, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8886835703592815, + "pass@2": 0.9419457099580372, + "pass@4": 0.9674046631147154, + "pass@8": 0.9800132434000822, + "pass@16": 0.9872327356476097, + "pass@32": 0.9919673623743022, + "pass@64": 0.9938730559494572, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8057934864457831, + "pass@2": 0.8846412964851539, + "pass@4": 0.929690460096855, + "pass@8": 0.9521490248359574, + "pass@16": 0.9636872668525197, + "pass@32": 0.9720114728755084, + "pass@64": 0.9792833961748288, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.6970023510971787, + "pass@2": 0.7992438383975512, + "pass@4": 0.8650445735819062, + "pass@8": 0.9031356523629713, + "pass@16": 0.9249747945390557, + "pass@32": 0.9407246495295145, + "pass@64": 0.9545253669368997, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.48713235294117646, + "pass@2": 0.6071869692758992, + "pass@4": 0.7042174234523209, + "pass@8": 0.775814783223066, + "pass@16": 0.8292051212942726, + "pass@32": 0.8707959721079798, + "pass@64": 0.9030591892971792, + "pass@128": 0.9299719887955182 + }, + "14": { + "pass@1": 0.48384959795321636, + "pass@2": 0.5952749430169916, + "pass@4": 0.6884485705734146, + "pass@8": 0.7624990556025817, + "pass@16": 0.8195176330850231, + "pass@32": 0.86363662023738, + "pass@64": 0.8976129475131868, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.3916903409090909, + "pass@2": 0.5208993229539489, + "pass@4": 0.6331746480553567, + "pass@8": 0.7205971077124683, + "pass@16": 0.7829411999553293, + "pass@32": 0.8238903194155507, + "pass@64": 0.8522462435594557, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.30088141025641024, + "pass@2": 0.4054595839782847, + "pass@4": 0.500781167311351, + "pass@8": 0.5809003310690419, + "pass@16": 0.6454340647690915, + "pass@32": 0.6993132560546717, + "pass@64": 0.7482662812431827, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.27986704192546585, + "pass@2": 0.3884398536460114, + "pass@4": 0.49235009938586827, + "pass@8": 0.583732081509763, + "pass@16": 0.6622393198782099, + "pass@32": 0.7289934318444642, + "pass@64": 0.7808316991010685, + "pass@128": 0.8167701863354038 + }, + "18": { + "pass@1": 0.26505055147058826, + "pass@2": 0.37142195460861493, + "pass@4": 0.47396713728798606, + "pass@8": 0.5602206902178573, + "pass@16": 0.6280354105963233, + "pass@32": 0.6817861507678503, + "pass@64": 0.7276384154090412, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.19022042410714285, + "pass@2": 0.26708416818991365, + "pass@4": 0.34927864904163775, + "pass@8": 0.4289785213982712, + "pass@16": 0.501354952736681, + "pass@32": 0.565596716464881, + "pass@64": 0.619600156910572, + "pass@128": 0.6607142857142857 + }, + "20": { + "pass@1": 0.18013192365269462, + "pass@2": 0.2592645663397615, + "pass@4": 0.34027802999176016, + "pass@8": 0.4190993109617173, + "pass@16": 0.4949920405942872, + "pass@32": 0.5656735198870328, + "pass@64": 0.627234567640334, + "pass@128": 0.6796407185628742 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 534291, + "answer_accuracy": 0.6669034090909091, + "avg_response_len": 299.6649474756351, + "resp_tokens_sum": 240077172, + "pass_at_k": { + "pass@1": 0.6669034090909091, + "pass@2": 0.731048707184485, + "pass@4": 0.7826457134721821, + "pass@8": 0.8233838836513981, + "pass@16": 0.855424083910444, + "pass@32": 0.8812548858982834, + "pass@64": 0.9028641749091145, + "pass@128": 0.9210736539383289 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.853008395522388, + "pass@2": 0.9105201110588794, + "pass@4": 0.9452996695375757, + "pass@8": 0.9676232015937645, + "pass@16": 0.9817453487262422, + "pass@32": 0.9899036907312646, + "pass@64": 0.9952647158697007, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9567410714285715, + "pass@2": 0.9686216957255346, + "pass@4": 0.9776913940221758, + "pass@8": 0.9832660892002095, + "pass@16": 0.987077038751852, + "pass@32": 0.9913104686469809, + "pass@64": 0.9951026628988415, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9899553571428571, + "pass@2": 0.9944693217556989, + "pass@4": 0.9968871366480065, + "pass@8": 0.9982643644219408, + "pass@16": 0.9991564768381861, + "pass@32": 0.9998017057315063, + "pass@64": 0.9999957879397594, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9876187865497076, + "pass@2": 0.992061230832988, + "pass@4": 0.9941773188439165, + "pass@8": 0.995596294224904, + "pass@16": 0.9970248009765742, + "pass@32": 0.9986489076449245, + "pass@64": 0.9998314684348667, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9397518382352941, + "pass@2": 0.9616778601204263, + "pass@4": 0.9757145073777543, + "pass@8": 0.9849050022589428, + "pass@16": 0.9905917398194917, + "pass@32": 0.9937739348835954, + "pass@64": 0.9960693668366222, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9750098580441641, + "pass@2": 0.9878835322536579, + "pass@4": 0.9931930843029477, + "pass@8": 0.9959053815542414, + "pass@16": 0.9972602517091385, + "pass@32": 0.9982078268425956, + "pass@64": 0.9992175497484641, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9476611946202531, + "pass@2": 0.9664474421907704, + "pass@4": 0.9756522099096789, + "pass@8": 0.9829886564110945, + "pass@16": 0.9887713880438683, + "pass@32": 0.9929847168303246, + "pass@64": 0.9958242851064026, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9275290915915916, + "pass@2": 0.9633303332860415, + "pass@4": 0.9809602068322539, + "pass@8": 0.990877053476471, + "pass@16": 0.9958726441061623, + "pass@32": 0.998631778027631, + "pass@64": 0.9998844176036991, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9133842443729904, + "pass@2": 0.9578445040129633, + "pass@4": 0.9793850611921092, + "pass@8": 0.9896856833560922, + "pass@16": 0.99553284800179, + "pass@32": 0.9990190461253682, + "pass@64": 0.9999781623934679, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8161706349206349, + "pass@2": 0.8948475190601176, + "pass@4": 0.9384380973211678, + "pass@8": 0.9608757978792809, + "pass@16": 0.9722872838437171, + "pass@32": 0.9792124678492932, + "pass@64": 0.9842550423043461, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.6806431574923547, + "pass@2": 0.7891289069084256, + "pass@4": 0.8598583147290075, + "pass@8": 0.9031737776318244, + "pass@16": 0.9307941653752537, + "pass@32": 0.9500838276534974, + "pass@64": 0.9648871426463134, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5169270833333334, + "pass@2": 0.6433844749441041, + "pass@4": 0.7417766948691599, + "pass@8": 0.8131684681108555, + "pass@16": 0.8627189914666871, + "pass@32": 0.895405386300501, + "pass@64": 0.9191195382859483, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.48516320474777447, + "pass@2": 0.5998584955255963, + "pass@4": 0.6965931077050082, + "pass@8": 0.7726242956261022, + "pass@16": 0.8291613907224347, + "pass@32": 0.8692043952342279, + "pass@64": 0.8986791706290098, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.43940489969135804, + "pass@2": 0.5578529029357437, + "pass@4": 0.6548958506922737, + "pass@8": 0.7317516466486336, + "pass@16": 0.7917271528246511, + "pass@32": 0.8386735973109505, + "pass@64": 0.8741588847793955, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.2929341814159292, + "pass@2": 0.3967725826307111, + "pass@4": 0.4962346490427633, + "pass@8": 0.5828059998718874, + "pass@16": 0.6554737344032179, + "pass@32": 0.7187961608827936, + "pass@64": 0.7760763588608376, + "pass@128": 0.8289085545722714 + }, + "17": { + "pass@1": 0.26418957078313254, + "pass@2": 0.36834629363912336, + "pass@4": 0.46793826797629223, + "pass@8": 0.551661206823551, + "pass@16": 0.6195107429620204, + "pass@32": 0.678079370980133, + "pass@64": 0.733369052164855, + "pass@128": 0.786144578313253 + }, + "18": { + "pass@1": 0.2745463709677419, + "pass@2": 0.36830907099314175, + "pass@4": 0.4597118479141718, + "pass@8": 0.5431559084782237, + "pass@16": 0.6148560345069214, + "pass@32": 0.6740668401066608, + "pass@64": 0.7212544085035065, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.1945100075528701, + "pass@2": 0.2777523372267288, + "pass@4": 0.36351960772652664, + "pass@8": 0.4440731680580169, + "pass@16": 0.5174175968867194, + "pass@32": 0.5841877385955095, + "pass@64": 0.6443669815167831, + "pass@128": 0.6948640483383686 + }, + "20": { + "pass@1": 0.1973066816816817, + "pass@2": 0.27872212763945037, + "pass@4": 0.363847763961937, + "pass@8": 0.4455984222743029, + "pass@16": 0.5212068839225962, + "pass@32": 0.5897195240458061, + "pass@64": 0.6530250083042405, + "pass@128": 0.7057057057057057 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f7cd23280582d248d023726f05a8eed4448967ff --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fdf70c1b8d08ae009ca23f7cac9aa0bd0cff26fa14f734d0d51607b4faf013d +size 2649830426 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/summary.csv b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..89f12fe1b7c4ae518225d4f5895b39ae121356c5 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op17-20,0.9438151041666667,0.4157703125,0.6658967927631579,0.0,0.0,0.0,201.10614409722223,354.7530515625,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/summary.json b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..20d11448c50bd471cb85bcd13699041f0c0cb36e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op17-20", + "id_acc": 0.9438151041666667, + "ood_acc": 0.4157703125, + "total_acc": 0.6658967927631579, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 201.10614409722223, + "ood_avg_resp_len": 354.7530515625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 540802, + "answer_accuracy": 0.6678810662345874, + "avg_response_len": 268.15522866938034, + "resp_tokens_sum": 217132797, + "pass_at_k": { + "pass@1": 0.6678810662345874, + "pass@2": 0.731164473078238, + "pass@4": 0.779814620602667, + "pass@8": 0.8176583751887917, + "pass@16": 0.8474172445850535, + "pass@32": 0.870803790094637, + "pass@64": 0.8902462938136585, + "pass@128": 0.9072083465064812 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8778576696165191, + "pass@2": 0.9216935376164258, + "pass@4": 0.9474172199713969, + "pass@8": 0.9654188468359517, + "pass@16": 0.9777636401292872, + "pass@32": 0.9863328110936131, + "pass@64": 0.9917817546869228, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9557778426791277, + "pass@2": 0.9654906235435523, + "pass@4": 0.973800176263014, + "pass@8": 0.9798030323439492, + "pass@16": 0.9832815566173658, + "pass@32": 0.9858820824757502, + "pass@64": 0.9882787447648765, + "pass@128": 0.9906542056074766 + }, + "3": { + "pass@1": 0.9922591743119266, + "pass@2": 0.9963278672734716, + "pass@4": 0.9980850739414453, + "pass@8": 0.9992066793724687, + "pass@16": 0.9998328881169319, + "pass@32": 0.9999944749477707, + "pass@64": 0.9999999994985725, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.982568359375, + "pass@2": 0.9903170367864174, + "pass@4": 0.9939585638709225, + "pass@8": 0.9956336116662886, + "pass@16": 0.9965011897205394, + "pass@32": 0.9968440691138198, + "pass@64": 0.9968749622828345, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9425595238095238, + "pass@2": 0.9607877140357454, + "pass@4": 0.9715392290249432, + "pass@8": 0.9791186410445809, + "pass@16": 0.9853992284800553, + "pass@32": 0.9910054778381431, + "pass@64": 0.995247280989616, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9483065119760479, + "pass@2": 0.9705814748455845, + "pass@4": 0.9804762084380167, + "pass@8": 0.9862780220628213, + "pass@16": 0.9896832084934538, + "pass@32": 0.9908509464056335, + "pass@64": 0.9910159624491164, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9368296967455622, + "pass@2": 0.9643732819270364, + "pass@4": 0.9772801360258956, + "pass@8": 0.9849902895915007, + "pass@16": 0.9902276572440275, + "pass@32": 0.9929242140601698, + "pass@64": 0.9939053564147182, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.9312650240384616, + "pass@2": 0.9645184263325259, + "pass@4": 0.9803785148851585, + "pass@8": 0.9896741609551327, + "pass@16": 0.9946263420996423, + "pass@32": 0.9971043502224575, + "pass@64": 0.9983853532572591, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9206646126760564, + "pass@2": 0.9607331984030167, + "pass@4": 0.9777831326541929, + "pass@8": 0.986769595696414, + "pass@16": 0.9927609271921384, + "pass@32": 0.9964592233943435, + "pass@64": 0.9984671060065928, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8213748229461756, + "pass@2": 0.8935474336954341, + "pass@4": 0.9340763818297493, + "pass@8": 0.9559787207718502, + "pass@16": 0.9685268552496731, + "pass@32": 0.9767709530477564, + "pass@64": 0.982877126664917, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.6984463276836158, + "pass@2": 0.8106034382089947, + "pass@4": 0.8795497037976184, + "pass@8": 0.9196428046531496, + "pass@16": 0.9460275011724377, + "pass@32": 0.9631035966250991, + "pass@64": 0.9734107084727417, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.5377644984326019, + "pass@2": 0.6605780860464542, + "pass@4": 0.7524930616039762, + "pass@8": 0.8165367240179848, + "pass@16": 0.8641324613574773, + "pass@32": 0.9023888533468492, + "pass@64": 0.9334225133172553, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.5067659657320872, + "pass@2": 0.6147987342703658, + "pass@4": 0.6982890669624235, + "pass@8": 0.7658124058064986, + "pass@16": 0.8180146824253777, + "pass@32": 0.8548311069597581, + "pass@64": 0.8841931928289547, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.4333227962427746, + "pass@2": 0.5478261828091572, + "pass@4": 0.6419825939387632, + "pass@8": 0.7153113400084069, + "pass@16": 0.7722291645122227, + "pass@32": 0.8171855305285305, + "pass@64": 0.854938200858591, + "pass@128": 0.8872832369942196 + }, + "16": { + "pass@1": 0.3077620967741935, + "pass@2": 0.4179562484124969, + "pass@4": 0.5183253746507492, + "pass@8": 0.5993537141790105, + "pass@16": 0.6641207838509781, + "pass@32": 0.7170970581981619, + "pass@64": 0.7597654177085406, + "pass@128": 0.7935483870967742 + }, + "17": { + "pass@1": 0.2698022037572254, + "pass@2": 0.36404415195029793, + "pass@4": 0.45026019336094525, + "pass@8": 0.5247725639243038, + "pass@16": 0.5872757336442391, + "pass@32": 0.6408713150622185, + "pass@64": 0.6923314852463324, + "pass@128": 0.7427745664739884 + }, + "18": { + "pass@1": 0.2632589285714286, + "pass@2": 0.3572957677165353, + "pass@4": 0.4467472826610956, + "pass@8": 0.5262610157224527, + "pass@16": 0.589722143816733, + "pass@32": 0.6356788942331097, + "pass@64": 0.6732118806686305, + "pass@128": 0.7085714285714285 + }, + "19": { + "pass@1": 0.19390484234234234, + "pass@2": 0.2684835425977157, + "pass@4": 0.34286924128853263, + "pass@8": 0.4151729621852111, + "pass@16": 0.48393435094925585, + "pass@32": 0.5479942111320676, + "pass@64": 0.6062296128855615, + "pass@128": 0.6606606606606606 + }, + "20": { + "pass@1": 0.20190503003003002, + "pass@2": 0.286834447636613, + "pass@4": 0.37134299935481024, + "pass@8": 0.4471309689317958, + "pass@16": 0.5122863221912953, + "pass@32": 0.5662567887278481, + "pass@64": 0.6136344244399008, + "pass@128": 0.6576576576576577 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 544368, + "answer_accuracy": 0.6629579111457521, + "avg_response_len": 278.3371675272798, + "resp_tokens_sum": 228548215, + "pass_at_k": { + "pass@1": 0.6629579111457521, + "pass@2": 0.7276347918878587, + "pass@4": 0.7792220371177242, + "pass@8": 0.8196055261305205, + "pass@16": 0.8513215095014183, + "pass@32": 0.8768755383415355, + "pass@64": 0.8975920505534613, + "pass@128": 0.9151987529228371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8696798312883436, + "pass@2": 0.9186146653543302, + "pass@4": 0.9464048984866151, + "pass@8": 0.9644575325567745, + "pass@16": 0.9772704068767525, + "pass@32": 0.9862547560604297, + "pass@64": 0.9922762044109446, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9586103723404256, + "pass@2": 0.9672702564440083, + "pass@4": 0.9736437885423895, + "pass@8": 0.9779361963612356, + "pass@16": 0.9810863329863315, + "pass@32": 0.9836883585763785, + "pass@64": 0.9859087861062281, + "pass@128": 0.9878419452887538 + }, + "3": { + "pass@1": 0.9908617424242424, + "pass@2": 0.9960514346218085, + "pass@4": 0.9976872544341046, + "pass@8": 0.9984464684143097, + "pass@16": 0.9992439929783652, + "pass@32": 0.9998493181625818, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9850684171597633, + "pass@2": 0.9900158848949354, + "pass@4": 0.9916054067575871, + "pass@8": 0.9926058357250064, + "pass@16": 0.9935995492288452, + "pass@32": 0.994621258385564, + "pass@64": 0.9955578558086908, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.9645833333333333, + "pass@2": 0.9754475493552436, + "pass@4": 0.982398531976981, + "pass@8": 0.9883361791394907, + "pass@16": 0.9929284629109592, + "pass@32": 0.9956182396383584, + "pass@64": 0.9968886353429263, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9753313037249284, + "pass@2": 0.9916807441960164, + "pass@4": 0.9966340057242126, + "pass@8": 0.9980058334175127, + "pass@16": 0.9987355344474242, + "pass@32": 0.9995102342609697, + "pass@64": 0.9999603905476593, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9460350433526011, + "pass@2": 0.9703241631013609, + "pass@4": 0.9808562748298078, + "pass@8": 0.9870636604011149, + "pass@16": 0.9908352195978022, + "pass@32": 0.9934234406974602, + "pass@64": 0.9956567243821888, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9247579225352113, + "pass@2": 0.9658260092048359, + "pass@4": 0.9840326755810451, + "pass@8": 0.9923905005669404, + "pass@16": 0.9966280877479383, + "pass@32": 0.9989143053771641, + "pass@64": 0.9998999175263501, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8886835703592815, + "pass@2": 0.9419457099580372, + "pass@4": 0.9674046631147154, + "pass@8": 0.9800132434000822, + "pass@16": 0.9872327356476097, + "pass@32": 0.9919673623743022, + "pass@64": 0.9938730559494572, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8057934864457831, + "pass@2": 0.8846412964851539, + "pass@4": 0.929690460096855, + "pass@8": 0.9521490248359574, + "pass@16": 0.9636872668525197, + "pass@32": 0.9720114728755084, + "pass@64": 0.9792833961748288, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.6970023510971787, + "pass@2": 0.7992438383975512, + "pass@4": 0.8650445735819062, + "pass@8": 0.9031356523629713, + "pass@16": 0.9249747945390557, + "pass@32": 0.9407246495295145, + "pass@64": 0.9545253669368997, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.48713235294117646, + "pass@2": 0.6071869692758992, + "pass@4": 0.7042174234523209, + "pass@8": 0.775814783223066, + "pass@16": 0.8292051212942726, + "pass@32": 0.8707959721079798, + "pass@64": 0.9030591892971792, + "pass@128": 0.9299719887955182 + }, + "14": { + "pass@1": 0.48384959795321636, + "pass@2": 0.5952749430169916, + "pass@4": 0.6884485705734146, + "pass@8": 0.7624990556025817, + "pass@16": 0.8195176330850231, + "pass@32": 0.86363662023738, + "pass@64": 0.8976129475131868, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.3916903409090909, + "pass@2": 0.5208993229539489, + "pass@4": 0.6331746480553567, + "pass@8": 0.7205971077124683, + "pass@16": 0.7829411999553293, + "pass@32": 0.8238903194155507, + "pass@64": 0.8522462435594557, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.30088141025641024, + "pass@2": 0.4054595839782847, + "pass@4": 0.500781167311351, + "pass@8": 0.5809003310690419, + "pass@16": 0.6454340647690915, + "pass@32": 0.6993132560546717, + "pass@64": 0.7482662812431827, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.27986704192546585, + "pass@2": 0.3884398536460114, + "pass@4": 0.49235009938586827, + "pass@8": 0.583732081509763, + "pass@16": 0.6622393198782099, + "pass@32": 0.7289934318444642, + "pass@64": 0.7808316991010685, + "pass@128": 0.8167701863354038 + }, + "18": { + "pass@1": 0.26505055147058826, + "pass@2": 0.37142195460861493, + "pass@4": 0.47396713728798606, + "pass@8": 0.5602206902178573, + "pass@16": 0.6280354105963233, + "pass@32": 0.6817861507678503, + "pass@64": 0.7276384154090412, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.19022042410714285, + "pass@2": 0.26708416818991365, + "pass@4": 0.34927864904163775, + "pass@8": 0.4289785213982712, + "pass@16": 0.501354952736681, + "pass@32": 0.565596716464881, + "pass@64": 0.619600156910572, + "pass@128": 0.6607142857142857 + }, + "20": { + "pass@1": 0.18013192365269462, + "pass@2": 0.2592645663397615, + "pass@4": 0.34027802999176016, + "pass@8": 0.4190993109617173, + "pass@16": 0.4949920405942872, + "pass@32": 0.5656735198870328, + "pass@64": 0.627234567640334, + "pass@128": 0.6796407185628742 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 534291, + "answer_accuracy": 0.6669034090909091, + "avg_response_len": 299.6649474756351, + "resp_tokens_sum": 240077172, + "pass_at_k": { + "pass@1": 0.6669034090909091, + "pass@2": 0.731048707184485, + "pass@4": 0.7826457134721821, + "pass@8": 0.8233838836513981, + "pass@16": 0.855424083910444, + "pass@32": 0.8812548858982834, + "pass@64": 0.9028641749091145, + "pass@128": 0.9210736539383289 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.853008395522388, + "pass@2": 0.9105201110588794, + "pass@4": 0.9452996695375757, + "pass@8": 0.9676232015937645, + "pass@16": 0.9817453487262422, + "pass@32": 0.9899036907312646, + "pass@64": 0.9952647158697007, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9567410714285715, + "pass@2": 0.9686216957255346, + "pass@4": 0.9776913940221758, + "pass@8": 0.9832660892002095, + "pass@16": 0.987077038751852, + "pass@32": 0.9913104686469809, + "pass@64": 0.9951026628988415, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9899553571428571, + "pass@2": 0.9944693217556989, + "pass@4": 0.9968871366480065, + "pass@8": 0.9982643644219408, + "pass@16": 0.9991564768381861, + "pass@32": 0.9998017057315063, + "pass@64": 0.9999957879397594, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9876187865497076, + "pass@2": 0.992061230832988, + "pass@4": 0.9941773188439165, + "pass@8": 0.995596294224904, + "pass@16": 0.9970248009765742, + "pass@32": 0.9986489076449245, + "pass@64": 0.9998314684348667, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9397518382352941, + "pass@2": 0.9616778601204263, + "pass@4": 0.9757145073777543, + "pass@8": 0.9849050022589428, + "pass@16": 0.9905917398194917, + "pass@32": 0.9937739348835954, + "pass@64": 0.9960693668366222, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9750098580441641, + "pass@2": 0.9878835322536579, + "pass@4": 0.9931930843029477, + "pass@8": 0.9959053815542414, + "pass@16": 0.9972602517091385, + "pass@32": 0.9982078268425956, + "pass@64": 0.9992175497484641, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9476611946202531, + "pass@2": 0.9664474421907704, + "pass@4": 0.9756522099096789, + "pass@8": 0.9829886564110945, + "pass@16": 0.9887713880438683, + "pass@32": 0.9929847168303246, + "pass@64": 0.9958242851064026, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9275290915915916, + "pass@2": 0.9633303332860415, + "pass@4": 0.9809602068322539, + "pass@8": 0.990877053476471, + "pass@16": 0.9958726441061623, + "pass@32": 0.998631778027631, + "pass@64": 0.9998844176036991, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9133842443729904, + "pass@2": 0.9578445040129633, + "pass@4": 0.9793850611921092, + "pass@8": 0.9896856833560922, + "pass@16": 0.99553284800179, + "pass@32": 0.9990190461253682, + "pass@64": 0.9999781623934679, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8161706349206349, + "pass@2": 0.8948475190601176, + "pass@4": 0.9384380973211678, + "pass@8": 0.9608757978792809, + "pass@16": 0.9722872838437171, + "pass@32": 0.9792124678492932, + "pass@64": 0.9842550423043461, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.6806431574923547, + "pass@2": 0.7891289069084256, + "pass@4": 0.8598583147290075, + "pass@8": 0.9031737776318244, + "pass@16": 0.9307941653752537, + "pass@32": 0.9500838276534974, + "pass@64": 0.9648871426463134, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5169270833333334, + "pass@2": 0.6433844749441041, + "pass@4": 0.7417766948691599, + "pass@8": 0.8131684681108555, + "pass@16": 0.8627189914666871, + "pass@32": 0.895405386300501, + "pass@64": 0.9191195382859483, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.48516320474777447, + "pass@2": 0.5998584955255963, + "pass@4": 0.6965931077050082, + "pass@8": 0.7726242956261022, + "pass@16": 0.8291613907224347, + "pass@32": 0.8692043952342279, + "pass@64": 0.8986791706290098, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.43940489969135804, + "pass@2": 0.5578529029357437, + "pass@4": 0.6548958506922737, + "pass@8": 0.7317516466486336, + "pass@16": 0.7917271528246511, + "pass@32": 0.8386735973109505, + "pass@64": 0.8741588847793955, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.2929341814159292, + "pass@2": 0.3967725826307111, + "pass@4": 0.4962346490427633, + "pass@8": 0.5828059998718874, + "pass@16": 0.6554737344032179, + "pass@32": 0.7187961608827936, + "pass@64": 0.7760763588608376, + "pass@128": 0.8289085545722714 + }, + "17": { + "pass@1": 0.26418957078313254, + "pass@2": 0.36834629363912336, + "pass@4": 0.46793826797629223, + "pass@8": 0.551661206823551, + "pass@16": 0.6195107429620204, + "pass@32": 0.678079370980133, + "pass@64": 0.733369052164855, + "pass@128": 0.786144578313253 + }, + "18": { + "pass@1": 0.2745463709677419, + "pass@2": 0.36830907099314175, + "pass@4": 0.4597118479141718, + "pass@8": 0.5431559084782237, + "pass@16": 0.6148560345069214, + "pass@32": 0.6740668401066608, + "pass@64": 0.7212544085035065, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.1945100075528701, + "pass@2": 0.2777523372267288, + "pass@4": 0.36351960772652664, + "pass@8": 0.4440731680580169, + "pass@16": 0.5174175968867194, + "pass@32": 0.5841877385955095, + "pass@64": 0.6443669815167831, + "pass@128": 0.6948640483383686 + }, + "20": { + "pass@1": 0.1973066816816817, + "pass@2": 0.27872212763945037, + "pass@4": 0.363847763961937, + "pass@8": 0.4455984222743029, + "pass@16": 0.5212068839225962, + "pass@32": 0.5897195240458061, + "pass@64": 0.6530250083042405, + "pass@128": 0.7057057057057057 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4b10966929630870846b5118648a10ead27e930f --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebb8c35d2053d67fa9663c7c2914e3978d7bed4e24c26ca6c27044bca3bc6137 +size 1790645111 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bc7032a097936fdc5a44ecb575aa0cae992fbcfb --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9767369791666667, + "avg_response_len": 199.35722829861112, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9767369791666667, + "pass@2": 0.9847307524059445, + "pass@4": 0.9895344359038449, + "pass@8": 0.9927958069347775, + "pass@16": 0.9950750885460871, + "pass@32": 0.9966298612070215, + "pass@64": 0.997668799282402, + "pass@128": 0.9983333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.920578125, + "pass@2": 0.943384473425197, + "pass@4": 0.9593121506374203, + "pass@8": 0.9717550812683087, + "pass@16": 0.9810648927549185, + "pass@32": 0.9878354764388295, + "pass@64": 0.9926341307554416, + "pass@128": 0.996 + }, + "2": { + "pass@1": 0.9977421875, + "pass@2": 0.9987677165354332, + "pass@4": 0.9994533777652793, + "pass@8": 0.9998447844576749, + "pass@16": 0.9999827737314505, + "pass@32": 0.9999998583450264, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9968828125, + "pass@2": 0.9987525836614173, + "pass@4": 0.999656565335583, + "pass@8": 0.9999433308852453, + "pass@16": 0.9999976990491554, + "pass@32": 0.9999999983745457, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9988515625, + "pass@2": 0.999843626968504, + "pass@4": 0.9999945582114736, + "pass@8": 0.9999999916586431, + "pass@16": 0.9999999999999937, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9683125, + "pass@2": 0.9761728592519686, + "pass@4": 0.9821097341582299, + "pass@8": 0.9867184919139875, + "pass@16": 0.9899895824649028, + "pass@32": 0.9921478749670114, + "pass@64": 0.9936559996928844, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.9812265625, + "pass@2": 0.9887455708661417, + "pass@4": 0.9923427086614175, + "pass@8": 0.9939538139105087, + "pass@16": 0.9948345088716389, + "pass@32": 0.9956369987548309, + "pass@64": 0.9964751060407421, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9837890625, + "pass@2": 0.9888052411417323, + "pass@4": 0.9921420545556804, + "pass@8": 0.9946501654885048, + "pass@16": 0.9964303437364922, + "pass@32": 0.9974413651351424, + "pass@64": 0.9979251712047299, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.97440625, + "pass@2": 0.9852653789370085, + "pass@4": 0.9914850434945629, + "pass@8": 0.9950130141650685, + "pass@16": 0.9972397298056225, + "pass@32": 0.9988167961486235, + "pass@64": 0.9997030444450445, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.96884375, + "pass@2": 0.9828393208661422, + "pass@4": 0.9893137303149602, + "pass@8": 0.9932835886650618, + "pass@16": 0.9961362665006018, + "pass@32": 0.9977903826991764, + "pass@64": 0.9986257414029938, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.920578125, + "2": 0.9977421875, + "3": 0.9968828125, + "4": 0.9988515625, + "5": 0.9683125, + "6": 0.9812265625, + "7": 0.9837890625, + "8": 0.97440625, + "9": 0.96884375 + }, + "per_op_avg_response_len": { + "10": 278.8357421875, + "2": 115.658953125, + "3": 144.5778828125, + "4": 159.33465625, + "5": 173.715453125, + "6": 198.7001328125, + "7": 223.1321796875, + "8": 242.42684375, + "9": 257.8332109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.8357421875, + "2": 115.658953125, + "3": 144.5778828125, + "4": 159.33465625, + "5": 173.715453125, + "6": 198.7001328125, + "7": 223.1321796875, + "8": 242.42684375, + "9": 257.8332109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 370091, + "answer_accuracy": 0.9764727921310368, + "avg_response_len": 189.7772025920297, + "resp_tokens_sum": 71927078, + "pass_at_k": { + "pass@1": 0.9764727921310368, + "pass@2": 0.984829688842087, + "pass@4": 0.9896694791950401, + "pass@8": 0.992842062781418, + "pass@16": 0.9950258388301352, + "pass@32": 0.9966006379636904, + "pass@64": 0.9977145967463034, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262075958702065, + "pass@2": 0.9472844517222959, + "pass@4": 0.9625736202222505, + "pass@8": 0.9752088532456417, + "pass@16": 0.9852964010743517, + "pass@32": 0.9931721203155969, + "pass@64": 0.9981947094097119, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9980042834890965, + "pass@2": 0.9989306608286114, + "pass@4": 0.9996450604305304, + "pass@8": 0.9999632386498842, + "pass@16": 0.9999997146475239, + "pass@32": 0.9999999999967102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99873375382263, + "pass@2": 0.9998532651881819, + "pass@4": 0.9999967518395063, + "pass@8": 0.999999998425688, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986083984375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999975270669292, + "pass@8": 0.999999999294473, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9691220238095238, + "pass@2": 0.9768946850393702, + "pass@4": 0.9819980389356091, + "pass@8": 0.9855672510861936, + "pass@16": 0.9879905446880726, + "pass@32": 0.9901358747592569, + "pass@64": 0.9919511402537153, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9704809131736527, + "pass@2": 0.982392130109859, + "pass@4": 0.9884757380626823, + "pass@8": 0.9908924950302588, + "pass@16": 0.9916951265830015, + "pass@32": 0.9923322012920733, + "pass@64": 0.9932693667782507, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9794979659763313, + "pass@2": 0.9866467146950566, + "pass@4": 0.9912996885004757, + "pass@8": 0.9941373517267905, + "pass@16": 0.9959246088715661, + "pass@32": 0.9968292360948416, + "pass@64": 0.9970371429393932, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9788912259615384, + "pass@2": 0.9877851806985668, + "pass@4": 0.9929099971277628, + "pass@8": 0.9958272162861633, + "pass@16": 0.9973291315524355, + "pass@32": 0.9981958424173555, + "pass@64": 0.9992050267305785, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9716989436619718, + "pass@2": 0.9855533298214487, + "pass@4": 0.9912314567193186, + "pass@8": 0.9946283210768073, + "pass@16": 0.9972499146804312, + "pass@32": 0.9987380276325823, + "pass@64": 0.9996558144749331, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382661, + "answer_accuracy": 0.9795344241480997, + "avg_response_len": 197.28146502293578, + "resp_tokens_sum": 77069188, + "pass_at_k": { + "pass@1": 0.9795344241480997, + "pass@2": 0.9868609053957139, + "pass@4": 0.990972127419197, + "pass@8": 0.99358057190487, + "pass@16": 0.9953473881693898, + "pass@32": 0.9965926543310326, + "pass@64": 0.9976901353897375, + "pass@128": 0.9986893840104849 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9239120015337423, + "pass@2": 0.9454351540988355, + "pass@4": 0.9593524645692291, + "pass@8": 0.9691033208480212, + "pass@16": 0.9760508641263232, + "pass@32": 0.9818746126405525, + "pass@64": 0.9878373267292176, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9984564969604863, + "pass@2": 0.9993156606753941, + "pass@4": 0.9998513137035683, + "pass@8": 0.999993739366535, + "pass@16": 0.9999999933961763, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969460227272727, + "pass@2": 0.9988092042471962, + "pass@4": 0.9997825655315814, + "pass@8": 0.9999933566104181, + "pass@16": 0.999999996438375, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9996724013418442, + "pass@4": 0.9999862412723556, + "pass@8": 0.999999975989384, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9799592391304348, + "pass@2": 0.984868909049412, + "pass@4": 0.9885561446123581, + "pass@8": 0.9915921962517327, + "pass@16": 0.993767491284291, + "pass@32": 0.9948652815852641, + "pass@64": 0.9956520789485371, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9882476719197708, + "pass@2": 0.9931144665297928, + "pass@4": 0.9947072929458316, + "pass@8": 0.9954162070318462, + "pass@16": 0.9963557763664397, + "pass@32": 0.9977304914249616, + "pass@64": 0.9992067323962728, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866329479768786, + "pass@2": 0.9905159073323926, + "pass@4": 0.9934451435983797, + "pass@8": 0.99603050505388, + "pass@16": 0.9979728178127817, + "pass@32": 0.9990894429436349, + "pass@64": 0.9998278630547335, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9754401408450705, + "pass@2": 0.9891212432072756, + "pass@4": 0.995878176671578, + "pass@8": 0.9984830423791722, + "pass@16": 0.9995299276902677, + "pass@32": 0.9999462568241625, + "pass@64": 0.9999998252073421, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9667617889221557, + "pass@2": 0.979590400891131, + "pass@4": 0.9860774356798212, + "pass@8": 0.9907092343878937, + "pass@16": 0.9937065287352138, + "pass@32": 0.9952315126113409, + "pass@64": 0.9962629619361241, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 372449, + "answer_accuracy": 0.9741405465349849, + "avg_response_len": 210.97479965266155, + "resp_tokens_sum": 80663261, + "pass_at_k": { + "pass@1": 0.9741405465349849, + "pass@2": 0.9824561699780416, + "pass@4": 0.987931591012624, + "pass@8": 0.9919481115043722, + "pass@16": 0.9948456844478647, + "pass@32": 0.9966968466134541, + "pass@64": 0.9976016000556832, + "pass@128": 0.9979912956143288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9116371268656717, + "pass@2": 0.937442340463039, + "pass@4": 0.9559725072612185, + "pass@8": 0.9708405893181536, + "pass@16": 0.9816621823448725, + "pass@32": 0.988235832629917, + "pass@64": 0.9916750917964882, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9968303571428572, + "pass@2": 0.9981032058492688, + "pass@4": 0.9989035175960147, + "pass@8": 0.9995961274270628, + "pass@16": 0.999951050006438, + "pass@32": 0.9999995952745215, + "pass@64": 0.9999999999993562, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9950573979591837, + "pass@2": 0.997648771263286, + "pass@4": 0.9992110229114947, + "pass@8": 0.9998411770221788, + "pass@16": 0.9999932951151359, + "pass@32": 0.9999999952610663, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999314692982456, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9557444852941176, + "pass@2": 0.9666801760074109, + "pass@4": 0.9756720058889695, + "pass@8": 0.9828397062205556, + "pass@16": 0.9880081658678815, + "pass@32": 0.9912545655027337, + "pass@64": 0.9932100684579969, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9848186119873817, + "pass@2": 0.9906298125884897, + "pass@4": 0.9938137757425434, + "pass@8": 0.9955692880639988, + "pass@16": 0.9964674152713848, + "pass@32": 0.9968142019431131, + "pass@64": 0.9968453877933975, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9852650316455697, + "pass@2": 0.9892409735373268, + "pass@4": 0.9916162663211403, + "pass@8": 0.993687297013313, + "pass@16": 0.9952823765021533, + "pass@32": 0.9962915698626211, + "pass@64": 0.9967916970704975, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9691019144144144, + "pass@2": 0.9787938775389561, + "pass@4": 0.9854665815984716, + "pass@8": 0.9905508787963346, + "pass@16": 0.99471446375741, + "pass@32": 0.9981945109364294, + "pass@64": 0.9998532677372303, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.967820538585209, + "pass@2": 0.983230530166848, + "pass@4": 0.9906003204583351, + "pass@8": 0.9945133453287406, + "pass@16": 0.997474489361695, + "pass@32": 0.9994567770978201, + "pass@64": 0.9999874533045885, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.364609375, + "avg_response_len": 321.7899109375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.364609375, + "pass@2": 0.4365432824803183, + "pass@4": 0.4983515059242584, + "pass@8": 0.5496308039951803, + "pass@16": 0.5922104837677016, + "pass@32": 0.6287445872983515, + "pass@64": 0.6612323839032767, + "pass@128": 0.6911 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8752265625, + "pass@2": 0.9178388287401577, + "pass@4": 0.9424449039182602, + "pass@8": 0.9576268693137503, + "pass@16": 0.9674835737613855, + "pass@32": 0.9739366158145486, + "pass@64": 0.9788878586572707, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.72859375, + "pass@2": 0.8105559793307089, + "pass@4": 0.8639075352455943, + "pass@8": 0.8989734681682713, + "pass@16": 0.9227623540503849, + "pass@32": 0.9406052579708968, + "pass@64": 0.9547476588447842, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.4793515625, + "pass@2": 0.5754499261811029, + "pass@4": 0.6516107148481438, + "pass@8": 0.7104221527448548, + "pass@16": 0.7566340338661215, + "pass@32": 0.7938961399898414, + "pass@64": 0.8242966828592772, + "pass@128": 0.851 + }, + "14": { + "pass@1": 0.390921875, + "pass@2": 0.4819666584645672, + "pass@4": 0.5591535456505439, + "pass@8": 0.620376441643782, + "pass@16": 0.6693972153352593, + "pass@32": 0.7104943228409171, + "pass@64": 0.7458668839126971, + "pass@128": 0.779 + }, + "15": { + "pass@1": 0.2734453125, + "pass@2": 0.3596272145669292, + "pass@4": 0.43821871091113546, + "pass@8": 0.5062454695152218, + "pass@16": 0.5634195944120015, + "pass@32": 0.6128790386707711, + "pass@64": 0.6575047119613867, + "pass@128": 0.698 + }, + "16": { + "pass@1": 0.2146015625, + "pass@2": 0.28161946358267764, + "pass@4": 0.34281378308961374, + "pass@8": 0.3957892272668447, + "pass@16": 0.4430827639918865, + "pass@32": 0.48809134053016595, + "pass@64": 0.5315334893674112, + "pass@128": 0.574 + }, + "17": { + "pass@1": 0.1945625, + "pass@2": 0.2633271407480316, + "pass@4": 0.32818491854143195, + "pass@8": 0.384928824211218, + "pass@16": 0.43419454484317127, + "pass@32": 0.47886065463004496, + "pass@64": 0.5199017024764674, + "pass@128": 0.558 + }, + "18": { + "pass@1": 0.1785078125, + "pass@2": 0.2427426181102363, + "pass@4": 0.30599354921259825, + "pass@8": 0.3640781766615723, + "pass@16": 0.41537930444284954, + "pass@32": 0.45949115536796736, + "pass@64": 0.4988951929303225, + "pass@128": 0.537 + }, + "19": { + "pass@1": 0.154828125, + "pass@2": 0.21421272145669293, + "pass@4": 0.27264435639295076, + "pass@8": 0.3258449292993792, + "pass@16": 0.37189560715407044, + "pass@32": 0.4108748757157111, + "pass@64": 0.4463274590854478, + "pass@128": 0.48 + }, + "20": { + "pass@1": 0.1560546875, + "pass@2": 0.21809227362204722, + "pass@4": 0.2785430414323208, + "pass@8": 0.33202248112688454, + "pass@16": 0.3778558458197516, + "pass@32": 0.41831647145265843, + "pass@64": 0.45436219893766294, + "pass@128": 0.484 + } + }, + "per_op_accuracy": { + "11": 0.8752265625, + "12": 0.72859375, + "13": 0.4793515625, + "14": 0.390921875, + "15": 0.2734453125, + "16": 0.2146015625, + "17": 0.1945625, + "18": 0.1785078125, + "19": 0.154828125, + "20": 0.1560546875 + }, + "per_op_avg_response_len": { + "11": 295.0157265625, + "12": 302.1407890625, + "13": 313.043640625, + "14": 324.6075078125, + "15": 327.6606484375, + "16": 329.500875, + "17": 333.9622421875, + "18": 331.391953125, + "19": 329.3776484375, + "20": 331.198078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 295.0157265625, + "12": 302.1407890625, + "13": 313.043640625, + "14": 324.6075078125, + "15": 327.6606484375, + "16": 329.500875, + "17": 333.9622421875, + "18": 331.391953125, + "19": 329.3776484375, + "20": 331.198078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 158311, + "answer_accuracy": 0.3675496842496285, + "avg_response_len": 305.53790397473995, + "resp_tokens_sum": 131601286, + "pass_at_k": { + "pass@1": 0.3675496842496285, + "pass@2": 0.4375549528495058, + "pass@4": 0.4976164789672471, + "pass@8": 0.5472162753527349, + "pass@16": 0.5879244359924969, + "pass@32": 0.6223167469851808, + "pass@64": 0.6524263172358917, + "pass@128": 0.6799405646359584 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8643767705382436, + "pass@2": 0.9103828823804963, + "pass@4": 0.9397818941187591, + "pass@8": 0.9583675060345839, + "pass@16": 0.9700929114863296, + "pass@32": 0.9764964629949469, + "pass@64": 0.9799056970942873, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.7268273305084746, + "pass@2": 0.8074880304951289, + "pass@4": 0.8605120331886477, + "pass@8": 0.8964947682973986, + "pass@16": 0.922687611092102, + "pass@32": 0.943056297458418, + "pass@64": 0.9612028509917379, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.49662029780564265, + "pass@2": 0.5931109304420804, + "pass@4": 0.6641063535749253, + "pass@8": 0.7155787477067609, + "pass@16": 0.7549038431626364, + "pass@32": 0.7884830266691044, + "pass@64": 0.8175602804808281, + "pass@128": 0.8401253918495298 + }, + "14": { + "pass@1": 0.371690031152648, + "pass@2": 0.45270339674246307, + "pass@4": 0.5231334678375484, + "pass@8": 0.5811583407810782, + "pass@16": 0.6307697432919271, + "pass@32": 0.6760275357891473, + "pass@64": 0.7145223586803378, + "pass@128": 0.7445482866043613 + }, + "15": { + "pass@1": 0.28312409682080925, + "pass@2": 0.3647250921669475, + "pass@4": 0.4396159986865799, + "pass@8": 0.5055135288446342, + "pass@16": 0.5624053295412474, + "pass@32": 0.6111659186934275, + "pass@64": 0.6532217114087345, + "pass@128": 0.6878612716763006 + }, + "16": { + "pass@1": 0.20592237903225807, + "pass@2": 0.2731811182372363, + "pass@4": 0.3374821392285641, + "pass@8": 0.39455191140565066, + "pass@16": 0.4439089193131919, + "pass@32": 0.4897533385182273, + "pass@64": 0.5339553882856808, + "pass@128": 0.5741935483870968 + }, + "17": { + "pass@1": 0.1907514450867052, + "pass@2": 0.25265584463611124, + "pass@4": 0.3085860812738003, + "pass@8": 0.35492782114498905, + "pass@16": 0.39348343300608796, + "pass@32": 0.4286547786529118, + "pass@64": 0.4619729230107275, + "pass@128": 0.49710982658959535 + }, + "18": { + "pass@1": 0.17484375, + "pass@2": 0.2366777980877389, + "pass@4": 0.29902547565482895, + "pass@8": 0.3578314599200364, + "pass@16": 0.4098129193012817, + "pass@32": 0.4533627306820471, + "pass@64": 0.4913254507266545, + "pass@128": 0.5314285714285715 + }, + "19": { + "pass@1": 0.15411505255255256, + "pass@2": 0.21568640490884577, + "pass@4": 0.27843260527118807, + "pass@8": 0.3359369419932084, + "pass@16": 0.3827028678353523, + "pass@32": 0.418045292893194, + "pass@64": 0.44838724505377353, + "pass@128": 0.4774774774774775 + }, + "20": { + "pass@1": 0.16917698948948948, + "pass@2": 0.233259957792438, + "pass@4": 0.29268785742998343, + "pass@8": 0.3416415277999142, + "pass@16": 0.38070945313753074, + "pass@32": 0.4130096512830508, + "pass@64": 0.43969446344959395, + "pass@128": 0.46546546546546547 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 154226, + "answer_accuracy": 0.35827850877192985, + "avg_response_len": 317.23864945732976, + "resp_tokens_sum": 136559818, + "pass_at_k": { + "pass@1": 0.35827850877192985, + "pass@2": 0.4295325490926048, + "pass@4": 0.49169765172640867, + "pass@8": 0.5436254215716934, + "pass@16": 0.5866167841651377, + "pass@32": 0.6239667569832025, + "pass@64": 0.6578644376308234, + "pass@128": 0.6895628902765388 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8809299698795181, + "pass@2": 0.9194373458400537, + "pass@4": 0.9411916425544699, + "pass@8": 0.9545487908977287, + "pass@16": 0.9626235409000866, + "pass@32": 0.9688801959881596, + "pass@64": 0.9750893893190955, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.7426283307210031, + "pass@2": 0.8214161195171922, + "pass@4": 0.8708843078588523, + "pass@8": 0.9025153476243613, + "pass@16": 0.9236789138226168, + "pass@32": 0.9398016133683879, + "pass@64": 0.9512363959324338, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.44568452380952384, + "pass@2": 0.5398945995721127, + "pass@4": 0.6209990255944895, + "pass@8": 0.6893619895760472, + "pass@16": 0.7460630135915871, + "pass@32": 0.7918121601901343, + "pass@64": 0.8266485140373896, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.3906935307017544, + "pass@2": 0.4798242736105354, + "pass@4": 0.5533021609689135, + "pass@8": 0.610189362344645, + "pass@16": 0.655899075225643, + "pass@32": 0.6963984026027993, + "pass@64": 0.734174599789174, + "pass@128": 0.7748538011695907 + }, + "15": { + "pass@1": 0.23965435606060606, + "pass@2": 0.32455708661417326, + "pass@4": 0.40661831504016543, + "pass@8": 0.47953448478625676, + "pass@16": 0.5390197751191101, + "pass@32": 0.5891773536829282, + "pass@64": 0.6362165742536966, + "pass@128": 0.6818181818181818 + }, + "16": { + "pass@1": 0.21923967236467237, + "pass@2": 0.28664480842138323, + "pass@4": 0.34611463497618344, + "pass@8": 0.39435134301166086, + "pass@16": 0.4344641369924253, + "pass@32": 0.47232332153629814, + "pass@64": 0.5126976898405665, + "pass@128": 0.5584045584045584 + }, + "17": { + "pass@1": 0.20615780279503104, + "pass@2": 0.28085950872988696, + "pass@4": 0.3537120633600925, + "pass@8": 0.4190615788920885, + "pass@16": 0.4747636461754707, + "pass@32": 0.5222883827191508, + "pass@64": 0.5621582601092799, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.17676930147058822, + "pass@2": 0.24354989289022702, + "pass@4": 0.3090393573965019, + "pass@8": 0.367779967747244, + "pass@16": 0.4192643302519299, + "pass@32": 0.46349945288705857, + "pass@64": 0.5030580827068063, + "pass@128": 0.5411764705882353 + }, + "19": { + "pass@1": 0.15590122767857142, + "pass@2": 0.21318721878515182, + "pass@4": 0.2684598802939811, + "pass@8": 0.31754432950504796, + "pass@16": 0.3607783229282108, + "pass@32": 0.4025381677000241, + "pass@64": 0.44503200389778724, + "pass@128": 0.48214285714285715 + }, + "20": { + "pass@1": 0.14338510479041916, + "pass@2": 0.20299386463293878, + "pass@4": 0.26204798052938, + "pass@8": 0.3151829118182685, + "pass@16": 0.3618771130181657, + "pass@32": 0.40376215264946563, + "pass@64": 0.4417034842332708, + "pass@128": 0.47305389221556887 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 154163, + "answer_accuracy": 0.3680924319987775, + "avg_response_len": 343.1816883786675, + "resp_tokens_sum": 143729982, + "pass_at_k": { + "pass@1": 0.3680924319987775, + "pass@2": 0.44270857147498344, + "pass@4": 0.5059463339736895, + "pass@8": 0.5582863632775743, + "pass@16": 0.6023676239042687, + "pass@32": 0.6402658360769682, + "pass@64": 0.673750359957666, + "pass@128": 0.7041564792176039 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8813740079365079, + "pass@2": 0.9245094363204603, + "pass@4": 0.9467500633849341, + "pass@8": 0.9600410828745914, + "pass@16": 0.9696817791361351, + "pass@32": 0.9763972676484557, + "pass@64": 0.9817506994572943, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.7168147935779816, + "pass@2": 0.8032827963591705, + "pass@4": 0.860777312812962, + "pass@8": 0.8982016033603121, + "pass@16": 0.9219491321540287, + "pass@32": 0.9387358226486271, + "pass@64": 0.9511848296368877, + "pass@128": 0.9602446483180428 + }, + "13": { + "pass@1": 0.4994454089506173, + "pass@2": 0.5972381707494899, + "pass@4": 0.6730374565447833, + "pass@8": 0.7285503146535492, + "pass@16": 0.7699852223612504, + "pass@32": 0.8015219546127118, + "pass@64": 0.8283377588720496, + "pass@128": 0.8580246913580247 + }, + "14": { + "pass@1": 0.4094723664688427, + "pass@2": 0.5120147374938671, + "pass@4": 0.5994016718792938, + "pass@8": 0.6680707784308229, + "pass@16": 0.7198891513690838, + "pass@32": 0.7576297630933031, + "pass@64": 0.7875890019240686, + "pass@128": 0.8160237388724035 + }, + "15": { + "pass@1": 0.2975260416666667, + "pass@2": 0.38990275955088916, + "pass@4": 0.4689121339577924, + "pass@8": 0.5342327424552892, + "pass@16": 0.5893543969179779, + "pass@32": 0.6388490866897505, + "pass@64": 0.6837608960192697, + "pass@128": 0.7253086419753086 + }, + "16": { + "pass@1": 0.21773598820058998, + "pass@2": 0.28413271142545243, + "pass@4": 0.3442716314221784, + "pass@8": 0.3984094788613569, + "pass@16": 0.4512509938656513, + "pass@32": 0.5028976983193929, + "pass@64": 0.5488213270348411, + "pass@128": 0.5899705014749262 + }, + "17": { + "pass@1": 0.18728821536144577, + "pass@2": 0.2574441466654016, + "pass@4": 0.3238519578877939, + "pass@8": 0.3830902701560233, + "pass@16": 0.4372752498631445, + "pass@32": 0.4890639818691892, + "pass@64": 0.5392894920468873, + "pass@128": 0.5843373493975904 + }, + "18": { + "pass@1": 0.18455141129032257, + "pass@2": 0.2487045974091947, + "pass@4": 0.3105201652212829, + "pass@8": 0.3670708924370873, + "pass@16": 0.41740293677982304, + "pass@32": 0.46201414725048817, + "pass@64": 0.5028759259860611, + "pass@128": 0.5387096774193548 + }, + "19": { + "pass@1": 0.15445619335347432, + "pass@2": 0.21377112722125746, + "pass@4": 0.2710688189089655, + "pass@8": 0.32411792417505997, + "pass@16": 0.37230826483691687, + "pass@32": 0.41212377291561847, + "pass@64": 0.44557025127759675, + "pass@128": 0.48036253776435045 + }, + "20": { + "pass@1": 0.155640015015015, + "pass@2": 0.21806833900829967, + "pass@4": 0.2809428210550256, + "pass@8": 0.33929357303967483, + "pass@16": 0.39102895548614497, + "pass@32": 0.43822131708853057, + "pass@64": 0.4817266632883948, + "pass@128": 0.5135135135135135 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6545645559210527, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6545645559210527, + "pass@2": 0.6962110313924389, + "pass@4": 0.7310171043356241, + "pass@8": 0.7595510685455242, + "pass@16": 0.7830410860311512, + "pass@32": 0.8030060328340529, + "pass@64": 0.8205970017144427, + "pass@128": 0.8366315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.920578125, + "pass@2": 0.943384473425197, + "pass@4": 0.9593121506374203, + "pass@8": 0.9717550812683087, + "pass@16": 0.9810648927549185, + "pass@32": 0.9878354764388295, + "pass@64": 0.9926341307554416, + "pass@128": 0.996 + }, + "2": { + "pass@1": 0.9977421875, + "pass@2": 0.9987677165354332, + "pass@4": 0.9994533777652793, + "pass@8": 0.9998447844576749, + "pass@16": 0.9999827737314505, + "pass@32": 0.9999998583450264, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9968828125, + "pass@2": 0.9987525836614173, + "pass@4": 0.999656565335583, + "pass@8": 0.9999433308852453, + "pass@16": 0.9999976990491554, + "pass@32": 0.9999999983745457, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9988515625, + "pass@2": 0.999843626968504, + "pass@4": 0.9999945582114736, + "pass@8": 0.9999999916586431, + "pass@16": 0.9999999999999937, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9683125, + "pass@2": 0.9761728592519686, + "pass@4": 0.9821097341582299, + "pass@8": 0.9867184919139875, + "pass@16": 0.9899895824649028, + "pass@32": 0.9921478749670114, + "pass@64": 0.9936559996928844, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.9812265625, + "pass@2": 0.9887455708661417, + "pass@4": 0.9923427086614175, + "pass@8": 0.9939538139105087, + "pass@16": 0.9948345088716389, + "pass@32": 0.9956369987548309, + "pass@64": 0.9964751060407421, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9837890625, + "pass@2": 0.9888052411417323, + "pass@4": 0.9921420545556804, + "pass@8": 0.9946501654885048, + "pass@16": 0.9964303437364922, + "pass@32": 0.9974413651351424, + "pass@64": 0.9979251712047299, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.97440625, + "pass@2": 0.9852653789370085, + "pass@4": 0.9914850434945629, + "pass@8": 0.9950130141650685, + "pass@16": 0.9972397298056225, + "pass@32": 0.9988167961486235, + "pass@64": 0.9997030444450445, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.96884375, + "pass@2": 0.9828393208661422, + "pass@4": 0.9893137303149602, + "pass@8": 0.9932835886650618, + "pass@16": 0.9961362665006018, + "pass@32": 0.9977903826991764, + "pass@64": 0.9986257414029938, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.8752265625, + "pass@2": 0.9178388287401577, + "pass@4": 0.9424449039182602, + "pass@8": 0.9576268693137503, + "pass@16": 0.9674835737613855, + "pass@32": 0.9739366158145486, + "pass@64": 0.9788878586572707, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.72859375, + "pass@2": 0.8105559793307089, + "pass@4": 0.8639075352455943, + "pass@8": 0.8989734681682713, + "pass@16": 0.9227623540503849, + "pass@32": 0.9406052579708968, + "pass@64": 0.9547476588447842, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.4793515625, + "pass@2": 0.5754499261811029, + "pass@4": 0.6516107148481438, + "pass@8": 0.7104221527448548, + "pass@16": 0.7566340338661215, + "pass@32": 0.7938961399898414, + "pass@64": 0.8242966828592772, + "pass@128": 0.851 + }, + "14": { + "pass@1": 0.390921875, + "pass@2": 0.4819666584645672, + "pass@4": 0.5591535456505439, + "pass@8": 0.620376441643782, + "pass@16": 0.6693972153352593, + "pass@32": 0.7104943228409171, + "pass@64": 0.7458668839126971, + "pass@128": 0.779 + }, + "15": { + "pass@1": 0.2734453125, + "pass@2": 0.3596272145669292, + "pass@4": 0.43821871091113546, + "pass@8": 0.5062454695152218, + "pass@16": 0.5634195944120015, + "pass@32": 0.6128790386707711, + "pass@64": 0.6575047119613867, + "pass@128": 0.698 + }, + "16": { + "pass@1": 0.2146015625, + "pass@2": 0.28161946358267764, + "pass@4": 0.34281378308961374, + "pass@8": 0.3957892272668447, + "pass@16": 0.4430827639918865, + "pass@32": 0.48809134053016595, + "pass@64": 0.5315334893674112, + "pass@128": 0.574 + }, + "17": { + "pass@1": 0.1945625, + "pass@2": 0.2633271407480316, + "pass@4": 0.32818491854143195, + "pass@8": 0.384928824211218, + "pass@16": 0.43419454484317127, + "pass@32": 0.47886065463004496, + "pass@64": 0.5199017024764674, + "pass@128": 0.558 + }, + "18": { + "pass@1": 0.1785078125, + "pass@2": 0.2427426181102363, + "pass@4": 0.30599354921259825, + "pass@8": 0.3640781766615723, + "pass@16": 0.41537930444284954, + "pass@32": 0.45949115536796736, + "pass@64": 0.4988951929303225, + "pass@128": 0.537 + }, + "19": { + "pass@1": 0.154828125, + "pass@2": 0.21421272145669293, + "pass@4": 0.27264435639295076, + "pass@8": 0.3258449292993792, + "pass@16": 0.37189560715407044, + "pass@32": 0.4108748757157111, + "pass@64": 0.4463274590854478, + "pass@128": 0.48 + }, + "20": { + "pass@1": 0.1560546875, + "pass@2": 0.21809227362204722, + "pass@4": 0.2785430414323208, + "pass@8": 0.33202248112688454, + "pass@16": 0.3778558458197516, + "pass@32": 0.41831647145265843, + "pass@64": 0.45436219893766294, + "pass@128": 0.484 + } + }, + "per_op_accuracy": { + "10": 0.920578125, + "2": 0.9977421875, + "3": 0.9968828125, + "4": 0.9988515625, + "5": 0.9683125, + "6": 0.9812265625, + "7": 0.9837890625, + "8": 0.97440625, + "9": 0.96884375, + "11": 0.8752265625, + "12": 0.72859375, + "13": 0.4793515625, + "14": 0.390921875, + "15": 0.2734453125, + "16": 0.2146015625, + "17": 0.1945625, + "18": 0.1785078125, + "19": 0.154828125, + "20": 0.1560546875 + }, + "per_op_avg_response_len": { + "10": 278.8357421875, + "2": 115.658953125, + "3": 144.5778828125, + "4": 159.33465625, + "5": 173.715453125, + "6": 198.7001328125, + "7": 223.1321796875, + "8": 242.42684375, + "9": 257.8332109375, + "11": 295.0157265625, + "12": 302.1407890625, + "13": 313.043640625, + "14": 324.6075078125, + "15": 327.6606484375, + "16": 329.500875, + "17": 333.9622421875, + "18": 331.391953125, + "19": 329.3776484375, + "20": 331.198078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.8357421875, + "2": 115.658953125, + "3": 144.5778828125, + "4": 159.33465625, + "5": 173.715453125, + "6": 198.7001328125, + "7": 223.1321796875, + "8": 242.42684375, + "9": 257.8332109375, + "11": 295.0157265625, + "12": 302.1407890625, + "13": 313.043640625, + "14": 324.6075078125, + "15": 327.6606484375, + "16": 329.500875, + "17": 333.9622421875, + "18": 331.391953125, + "19": 329.3776484375, + "20": 331.198078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 528402, + "answer_accuracy": 0.6525672818526715, + "avg_response_len": 251.3539904758141, + "resp_tokens_sum": 203528364, + "pass_at_k": { + "pass@1": 0.6525672818526715, + "pass@2": 0.6937169024660155, + "pass@4": 0.7279308851756712, + "pass@8": 0.7557995754754544, + "pass@16": 0.7784756933118622, + "pass@32": 0.7975071676597604, + "pass@64": 0.8140448116447364, + "pass@128": 0.8289598482453368 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262075958702065, + "pass@2": 0.9472844517222959, + "pass@4": 0.9625736202222505, + "pass@8": 0.9752088532456417, + "pass@16": 0.9852964010743517, + "pass@32": 0.9931721203155969, + "pass@64": 0.9981947094097119, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9980042834890965, + "pass@2": 0.9989306608286114, + "pass@4": 0.9996450604305304, + "pass@8": 0.9999632386498842, + "pass@16": 0.9999997146475239, + "pass@32": 0.9999999999967102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99873375382263, + "pass@2": 0.9998532651881819, + "pass@4": 0.9999967518395063, + "pass@8": 0.999999998425688, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986083984375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999975270669292, + "pass@8": 0.999999999294473, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9691220238095238, + "pass@2": 0.9768946850393702, + "pass@4": 0.9819980389356091, + "pass@8": 0.9855672510861936, + "pass@16": 0.9879905446880726, + "pass@32": 0.9901358747592569, + "pass@64": 0.9919511402537153, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9704809131736527, + "pass@2": 0.982392130109859, + "pass@4": 0.9884757380626823, + "pass@8": 0.9908924950302588, + "pass@16": 0.9916951265830015, + "pass@32": 0.9923322012920733, + "pass@64": 0.9932693667782507, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9794979659763313, + "pass@2": 0.9866467146950566, + "pass@4": 0.9912996885004757, + "pass@8": 0.9941373517267905, + "pass@16": 0.9959246088715661, + "pass@32": 0.9968292360948416, + "pass@64": 0.9970371429393932, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9788912259615384, + "pass@2": 0.9877851806985668, + "pass@4": 0.9929099971277628, + "pass@8": 0.9958272162861633, + "pass@16": 0.9973291315524355, + "pass@32": 0.9981958424173555, + "pass@64": 0.9992050267305785, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9716989436619718, + "pass@2": 0.9855533298214487, + "pass@4": 0.9912314567193186, + "pass@8": 0.9946283210768073, + "pass@16": 0.9972499146804312, + "pass@32": 0.9987380276325823, + "pass@64": 0.9996558144749331, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8643767705382436, + "pass@2": 0.9103828823804963, + "pass@4": 0.9397818941187591, + "pass@8": 0.9583675060345839, + "pass@16": 0.9700929114863296, + "pass@32": 0.9764964629949469, + "pass@64": 0.9799056970942873, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.7268273305084746, + "pass@2": 0.8074880304951289, + "pass@4": 0.8605120331886477, + "pass@8": 0.8964947682973986, + "pass@16": 0.922687611092102, + "pass@32": 0.943056297458418, + "pass@64": 0.9612028509917379, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.49662029780564265, + "pass@2": 0.5931109304420804, + "pass@4": 0.6641063535749253, + "pass@8": 0.7155787477067609, + "pass@16": 0.7549038431626364, + "pass@32": 0.7884830266691044, + "pass@64": 0.8175602804808281, + "pass@128": 0.8401253918495298 + }, + "14": { + "pass@1": 0.371690031152648, + "pass@2": 0.45270339674246307, + "pass@4": 0.5231334678375484, + "pass@8": 0.5811583407810782, + "pass@16": 0.6307697432919271, + "pass@32": 0.6760275357891473, + "pass@64": 0.7145223586803378, + "pass@128": 0.7445482866043613 + }, + "15": { + "pass@1": 0.28312409682080925, + "pass@2": 0.3647250921669475, + "pass@4": 0.4396159986865799, + "pass@8": 0.5055135288446342, + "pass@16": 0.5624053295412474, + "pass@32": 0.6111659186934275, + "pass@64": 0.6532217114087345, + "pass@128": 0.6878612716763006 + }, + "16": { + "pass@1": 0.20592237903225807, + "pass@2": 0.2731811182372363, + "pass@4": 0.3374821392285641, + "pass@8": 0.39455191140565066, + "pass@16": 0.4439089193131919, + "pass@32": 0.4897533385182273, + "pass@64": 0.5339553882856808, + "pass@128": 0.5741935483870968 + }, + "17": { + "pass@1": 0.1907514450867052, + "pass@2": 0.25265584463611124, + "pass@4": 0.3085860812738003, + "pass@8": 0.35492782114498905, + "pass@16": 0.39348343300608796, + "pass@32": 0.4286547786529118, + "pass@64": 0.4619729230107275, + "pass@128": 0.49710982658959535 + }, + "18": { + "pass@1": 0.17484375, + "pass@2": 0.2366777980877389, + "pass@4": 0.29902547565482895, + "pass@8": 0.3578314599200364, + "pass@16": 0.4098129193012817, + "pass@32": 0.4533627306820471, + "pass@64": 0.4913254507266545, + "pass@128": 0.5314285714285715 + }, + "19": { + "pass@1": 0.15411505255255256, + "pass@2": 0.21568640490884577, + "pass@4": 0.27843260527118807, + "pass@8": 0.3359369419932084, + "pass@16": 0.3827028678353523, + "pass@32": 0.418045292893194, + "pass@64": 0.44838724505377353, + "pass@128": 0.4774774774774775 + }, + "20": { + "pass@1": 0.16917698948948948, + "pass@2": 0.233259957792438, + "pass@4": 0.29268785742998343, + "pass@8": 0.3416415277999142, + "pass@16": 0.38070945313753074, + "pass@32": 0.4130096512830508, + "pass@64": 0.43969446344959395, + "pass@128": 0.46546546546546547 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 536887, + "answer_accuracy": 0.6538471843335931, + "avg_response_len": 260.1678268706157, + "resp_tokens_sum": 213629006, + "pass_at_k": { + "pass@1": 0.6538471843335931, + "pass@2": 0.6946870531357957, + "pass@4": 0.7292324451503152, + "pass@8": 0.7576960558377633, + "pass@16": 0.7810744308402775, + "pass@32": 0.8012472306707449, + "pass@64": 0.8195398904071645, + "pass@128": 0.8366328916601715 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9239120015337423, + "pass@2": 0.9454351540988355, + "pass@4": 0.9593524645692291, + "pass@8": 0.9691033208480212, + "pass@16": 0.9760508641263232, + "pass@32": 0.9818746126405525, + "pass@64": 0.9878373267292176, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9984564969604863, + "pass@2": 0.9993156606753941, + "pass@4": 0.9998513137035683, + "pass@8": 0.999993739366535, + "pass@16": 0.9999999933961763, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969460227272727, + "pass@2": 0.9988092042471962, + "pass@4": 0.9997825655315814, + "pass@8": 0.9999933566104181, + "pass@16": 0.999999996438375, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9996724013418442, + "pass@4": 0.9999862412723556, + "pass@8": 0.999999975989384, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9799592391304348, + "pass@2": 0.984868909049412, + "pass@4": 0.9885561446123581, + "pass@8": 0.9915921962517327, + "pass@16": 0.993767491284291, + "pass@32": 0.9948652815852641, + "pass@64": 0.9956520789485371, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9882476719197708, + "pass@2": 0.9931144665297928, + "pass@4": 0.9947072929458316, + "pass@8": 0.9954162070318462, + "pass@16": 0.9963557763664397, + "pass@32": 0.9977304914249616, + "pass@64": 0.9992067323962728, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866329479768786, + "pass@2": 0.9905159073323926, + "pass@4": 0.9934451435983797, + "pass@8": 0.99603050505388, + "pass@16": 0.9979728178127817, + "pass@32": 0.9990894429436349, + "pass@64": 0.9998278630547335, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9754401408450705, + "pass@2": 0.9891212432072756, + "pass@4": 0.995878176671578, + "pass@8": 0.9984830423791722, + "pass@16": 0.9995299276902677, + "pass@32": 0.9999462568241625, + "pass@64": 0.9999998252073421, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9667617889221557, + "pass@2": 0.979590400891131, + "pass@4": 0.9860774356798212, + "pass@8": 0.9907092343878937, + "pass@16": 0.9937065287352138, + "pass@32": 0.9952315126113409, + "pass@64": 0.9962629619361241, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.8809299698795181, + "pass@2": 0.9194373458400537, + "pass@4": 0.9411916425544699, + "pass@8": 0.9545487908977287, + "pass@16": 0.9626235409000866, + "pass@32": 0.9688801959881596, + "pass@64": 0.9750893893190955, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.7426283307210031, + "pass@2": 0.8214161195171922, + "pass@4": 0.8708843078588523, + "pass@8": 0.9025153476243613, + "pass@16": 0.9236789138226168, + "pass@32": 0.9398016133683879, + "pass@64": 0.9512363959324338, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.44568452380952384, + "pass@2": 0.5398945995721127, + "pass@4": 0.6209990255944895, + "pass@8": 0.6893619895760472, + "pass@16": 0.7460630135915871, + "pass@32": 0.7918121601901343, + "pass@64": 0.8266485140373896, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.3906935307017544, + "pass@2": 0.4798242736105354, + "pass@4": 0.5533021609689135, + "pass@8": 0.610189362344645, + "pass@16": 0.655899075225643, + "pass@32": 0.6963984026027993, + "pass@64": 0.734174599789174, + "pass@128": 0.7748538011695907 + }, + "15": { + "pass@1": 0.23965435606060606, + "pass@2": 0.32455708661417326, + "pass@4": 0.40661831504016543, + "pass@8": 0.47953448478625676, + "pass@16": 0.5390197751191101, + "pass@32": 0.5891773536829282, + "pass@64": 0.6362165742536966, + "pass@128": 0.6818181818181818 + }, + "16": { + "pass@1": 0.21923967236467237, + "pass@2": 0.28664480842138323, + "pass@4": 0.34611463497618344, + "pass@8": 0.39435134301166086, + "pass@16": 0.4344641369924253, + "pass@32": 0.47232332153629814, + "pass@64": 0.5126976898405665, + "pass@128": 0.5584045584045584 + }, + "17": { + "pass@1": 0.20615780279503104, + "pass@2": 0.28085950872988696, + "pass@4": 0.3537120633600925, + "pass@8": 0.4190615788920885, + "pass@16": 0.4747636461754707, + "pass@32": 0.5222883827191508, + "pass@64": 0.5621582601092799, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.17676930147058822, + "pass@2": 0.24354989289022702, + "pass@4": 0.3090393573965019, + "pass@8": 0.367779967747244, + "pass@16": 0.4192643302519299, + "pass@32": 0.46349945288705857, + "pass@64": 0.5030580827068063, + "pass@128": 0.5411764705882353 + }, + "19": { + "pass@1": 0.15590122767857142, + "pass@2": 0.21318721878515182, + "pass@4": 0.2684598802939811, + "pass@8": 0.31754432950504796, + "pass@16": 0.3607783229282108, + "pass@32": 0.4025381677000241, + "pass@64": 0.44503200389778724, + "pass@128": 0.48214285714285715 + }, + "20": { + "pass@1": 0.14338510479041916, + "pass@2": 0.20299386463293878, + "pass@4": 0.26204798052938, + "pass@8": 0.3151829118182685, + "pass@16": 0.3618771130181657, + "pass@32": 0.40376215264946563, + "pass@64": 0.4417034842332708, + "pass@128": 0.47305389221556887 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 526612, + "answer_accuracy": 0.6573184614155616, + "avg_response_len": 280.08822670354687, + "resp_tokens_sum": 224393243, + "pass_at_k": { + "pass@1": 0.6573184614155616, + "pass@2": 0.7002938209922582, + "pass@4": 0.7359655004180538, + "pass@8": 0.7652439670407067, + "pass@16": 0.7896710217064363, + "pass@32": 0.8103663998207761, + "pass@64": 0.8283027891273101, + "pass@128": 0.8443840869148427 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9116371268656717, + "pass@2": 0.937442340463039, + "pass@4": 0.9559725072612185, + "pass@8": 0.9708405893181536, + "pass@16": 0.9816621823448725, + "pass@32": 0.988235832629917, + "pass@64": 0.9916750917964882, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9968303571428572, + "pass@2": 0.9981032058492688, + "pass@4": 0.9989035175960147, + "pass@8": 0.9995961274270628, + "pass@16": 0.999951050006438, + "pass@32": 0.9999995952745215, + "pass@64": 0.9999999999993562, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9950573979591837, + "pass@2": 0.997648771263286, + "pass@4": 0.9992110229114947, + "pass@8": 0.9998411770221788, + "pass@16": 0.9999932951151359, + "pass@32": 0.9999999952610663, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999314692982456, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9557444852941176, + "pass@2": 0.9666801760074109, + "pass@4": 0.9756720058889695, + "pass@8": 0.9828397062205556, + "pass@16": 0.9880081658678815, + "pass@32": 0.9912545655027337, + "pass@64": 0.9932100684579969, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9848186119873817, + "pass@2": 0.9906298125884897, + "pass@4": 0.9938137757425434, + "pass@8": 0.9955692880639988, + "pass@16": 0.9964674152713848, + "pass@32": 0.9968142019431131, + "pass@64": 0.9968453877933975, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9852650316455697, + "pass@2": 0.9892409735373268, + "pass@4": 0.9916162663211403, + "pass@8": 0.993687297013313, + "pass@16": 0.9952823765021533, + "pass@32": 0.9962915698626211, + "pass@64": 0.9967916970704975, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9691019144144144, + "pass@2": 0.9787938775389561, + "pass@4": 0.9854665815984716, + "pass@8": 0.9905508787963346, + "pass@16": 0.99471446375741, + "pass@32": 0.9981945109364294, + "pass@64": 0.9998532677372303, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.967820538585209, + "pass@2": 0.983230530166848, + "pass@4": 0.9906003204583351, + "pass@8": 0.9945133453287406, + "pass@16": 0.997474489361695, + "pass@32": 0.9994567770978201, + "pass@64": 0.9999874533045885, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8813740079365079, + "pass@2": 0.9245094363204603, + "pass@4": 0.9467500633849341, + "pass@8": 0.9600410828745914, + "pass@16": 0.9696817791361351, + "pass@32": 0.9763972676484557, + "pass@64": 0.9817506994572943, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.7168147935779816, + "pass@2": 0.8032827963591705, + "pass@4": 0.860777312812962, + "pass@8": 0.8982016033603121, + "pass@16": 0.9219491321540287, + "pass@32": 0.9387358226486271, + "pass@64": 0.9511848296368877, + "pass@128": 0.9602446483180428 + }, + "13": { + "pass@1": 0.4994454089506173, + "pass@2": 0.5972381707494899, + "pass@4": 0.6730374565447833, + "pass@8": 0.7285503146535492, + "pass@16": 0.7699852223612504, + "pass@32": 0.8015219546127118, + "pass@64": 0.8283377588720496, + "pass@128": 0.8580246913580247 + }, + "14": { + "pass@1": 0.4094723664688427, + "pass@2": 0.5120147374938671, + "pass@4": 0.5994016718792938, + "pass@8": 0.6680707784308229, + "pass@16": 0.7198891513690838, + "pass@32": 0.7576297630933031, + "pass@64": 0.7875890019240686, + "pass@128": 0.8160237388724035 + }, + "15": { + "pass@1": 0.2975260416666667, + "pass@2": 0.38990275955088916, + "pass@4": 0.4689121339577924, + "pass@8": 0.5342327424552892, + "pass@16": 0.5893543969179779, + "pass@32": 0.6388490866897505, + "pass@64": 0.6837608960192697, + "pass@128": 0.7253086419753086 + }, + "16": { + "pass@1": 0.21773598820058998, + "pass@2": 0.28413271142545243, + "pass@4": 0.3442716314221784, + "pass@8": 0.3984094788613569, + "pass@16": 0.4512509938656513, + "pass@32": 0.5028976983193929, + "pass@64": 0.5488213270348411, + "pass@128": 0.5899705014749262 + }, + "17": { + "pass@1": 0.18728821536144577, + "pass@2": 0.2574441466654016, + "pass@4": 0.3238519578877939, + "pass@8": 0.3830902701560233, + "pass@16": 0.4372752498631445, + "pass@32": 0.4890639818691892, + "pass@64": 0.5392894920468873, + "pass@128": 0.5843373493975904 + }, + "18": { + "pass@1": 0.18455141129032257, + "pass@2": 0.2487045974091947, + "pass@4": 0.3105201652212829, + "pass@8": 0.3670708924370873, + "pass@16": 0.41740293677982304, + "pass@32": 0.46201414725048817, + "pass@64": 0.5028759259860611, + "pass@128": 0.5387096774193548 + }, + "19": { + "pass@1": 0.15445619335347432, + "pass@2": 0.21377112722125746, + "pass@4": 0.2710688189089655, + "pass@8": 0.32411792417505997, + "pass@16": 0.37230826483691687, + "pass@32": 0.41212377291561847, + "pass@64": 0.44557025127759675, + "pass@128": 0.48036253776435045 + }, + "20": { + "pass@1": 0.155640015015015, + "pass@2": 0.21806833900829967, + "pass@4": 0.2809428210550256, + "pass@8": 0.33929357303967483, + "pass@16": 0.39102895548614497, + "pass@32": 0.43822131708853057, + "pass@64": 0.4817266632883948, + "pass@128": 0.5135135135135135 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..32a2e7ad7db11f67fdd273e8cc163f8e4e614151 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3b17d766db22f66ec91c8eba158c5e27318af2129a2f4adff84c7ecfd2f7ad7 +size 2560032647 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/summary.csv b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..62b749f0b2571aa2be54baf160d98258ef11b48b --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op7-10,0.9767369791666667,0.364609375,0.6545645559210527,0.0,0.0,0.0,199.35722829861112,321.7899109375,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/summary.json b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b767f16ce06a3b6262023309cbec032833f2b45d --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op7-10", + "id_acc": 0.9767369791666667, + "ood_acc": 0.364609375, + "total_acc": 0.6545645559210527, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.35722829861112, + "ood_avg_resp_len": 321.7899109375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 528402, + "answer_accuracy": 0.6525672818526715, + "avg_response_len": 251.3539904758141, + "resp_tokens_sum": 203528364, + "pass_at_k": { + "pass@1": 0.6525672818526715, + "pass@2": 0.6937169024660155, + "pass@4": 0.7279308851756712, + "pass@8": 0.7557995754754544, + "pass@16": 0.7784756933118622, + "pass@32": 0.7975071676597604, + "pass@64": 0.8140448116447364, + "pass@128": 0.8289598482453368 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262075958702065, + "pass@2": 0.9472844517222959, + "pass@4": 0.9625736202222505, + "pass@8": 0.9752088532456417, + "pass@16": 0.9852964010743517, + "pass@32": 0.9931721203155969, + "pass@64": 0.9981947094097119, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9980042834890965, + "pass@2": 0.9989306608286114, + "pass@4": 0.9996450604305304, + "pass@8": 0.9999632386498842, + "pass@16": 0.9999997146475239, + "pass@32": 0.9999999999967102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99873375382263, + "pass@2": 0.9998532651881819, + "pass@4": 0.9999967518395063, + "pass@8": 0.999999998425688, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986083984375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999975270669292, + "pass@8": 0.999999999294473, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9691220238095238, + "pass@2": 0.9768946850393702, + "pass@4": 0.9819980389356091, + "pass@8": 0.9855672510861936, + "pass@16": 0.9879905446880726, + "pass@32": 0.9901358747592569, + "pass@64": 0.9919511402537153, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9704809131736527, + "pass@2": 0.982392130109859, + "pass@4": 0.9884757380626823, + "pass@8": 0.9908924950302588, + "pass@16": 0.9916951265830015, + "pass@32": 0.9923322012920733, + "pass@64": 0.9932693667782507, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9794979659763313, + "pass@2": 0.9866467146950566, + "pass@4": 0.9912996885004757, + "pass@8": 0.9941373517267905, + "pass@16": 0.9959246088715661, + "pass@32": 0.9968292360948416, + "pass@64": 0.9970371429393932, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9788912259615384, + "pass@2": 0.9877851806985668, + "pass@4": 0.9929099971277628, + "pass@8": 0.9958272162861633, + "pass@16": 0.9973291315524355, + "pass@32": 0.9981958424173555, + "pass@64": 0.9992050267305785, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9716989436619718, + "pass@2": 0.9855533298214487, + "pass@4": 0.9912314567193186, + "pass@8": 0.9946283210768073, + "pass@16": 0.9972499146804312, + "pass@32": 0.9987380276325823, + "pass@64": 0.9996558144749331, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8643767705382436, + "pass@2": 0.9103828823804963, + "pass@4": 0.9397818941187591, + "pass@8": 0.9583675060345839, + "pass@16": 0.9700929114863296, + "pass@32": 0.9764964629949469, + "pass@64": 0.9799056970942873, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.7268273305084746, + "pass@2": 0.8074880304951289, + "pass@4": 0.8605120331886477, + "pass@8": 0.8964947682973986, + "pass@16": 0.922687611092102, + "pass@32": 0.943056297458418, + "pass@64": 0.9612028509917379, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.49662029780564265, + "pass@2": 0.5931109304420804, + "pass@4": 0.6641063535749253, + "pass@8": 0.7155787477067609, + "pass@16": 0.7549038431626364, + "pass@32": 0.7884830266691044, + "pass@64": 0.8175602804808281, + "pass@128": 0.8401253918495298 + }, + "14": { + "pass@1": 0.371690031152648, + "pass@2": 0.45270339674246307, + "pass@4": 0.5231334678375484, + "pass@8": 0.5811583407810782, + "pass@16": 0.6307697432919271, + "pass@32": 0.6760275357891473, + "pass@64": 0.7145223586803378, + "pass@128": 0.7445482866043613 + }, + "15": { + "pass@1": 0.28312409682080925, + "pass@2": 0.3647250921669475, + "pass@4": 0.4396159986865799, + "pass@8": 0.5055135288446342, + "pass@16": 0.5624053295412474, + "pass@32": 0.6111659186934275, + "pass@64": 0.6532217114087345, + "pass@128": 0.6878612716763006 + }, + "16": { + "pass@1": 0.20592237903225807, + "pass@2": 0.2731811182372363, + "pass@4": 0.3374821392285641, + "pass@8": 0.39455191140565066, + "pass@16": 0.4439089193131919, + "pass@32": 0.4897533385182273, + "pass@64": 0.5339553882856808, + "pass@128": 0.5741935483870968 + }, + "17": { + "pass@1": 0.1907514450867052, + "pass@2": 0.25265584463611124, + "pass@4": 0.3085860812738003, + "pass@8": 0.35492782114498905, + "pass@16": 0.39348343300608796, + "pass@32": 0.4286547786529118, + "pass@64": 0.4619729230107275, + "pass@128": 0.49710982658959535 + }, + "18": { + "pass@1": 0.17484375, + "pass@2": 0.2366777980877389, + "pass@4": 0.29902547565482895, + "pass@8": 0.3578314599200364, + "pass@16": 0.4098129193012817, + "pass@32": 0.4533627306820471, + "pass@64": 0.4913254507266545, + "pass@128": 0.5314285714285715 + }, + "19": { + "pass@1": 0.15411505255255256, + "pass@2": 0.21568640490884577, + "pass@4": 0.27843260527118807, + "pass@8": 0.3359369419932084, + "pass@16": 0.3827028678353523, + "pass@32": 0.418045292893194, + "pass@64": 0.44838724505377353, + "pass@128": 0.4774774774774775 + }, + "20": { + "pass@1": 0.16917698948948948, + "pass@2": 0.233259957792438, + "pass@4": 0.29268785742998343, + "pass@8": 0.3416415277999142, + "pass@16": 0.38070945313753074, + "pass@32": 0.4130096512830508, + "pass@64": 0.43969446344959395, + "pass@128": 0.46546546546546547 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 536887, + "answer_accuracy": 0.6538471843335931, + "avg_response_len": 260.1678268706157, + "resp_tokens_sum": 213629006, + "pass_at_k": { + "pass@1": 0.6538471843335931, + "pass@2": 0.6946870531357957, + "pass@4": 0.7292324451503152, + "pass@8": 0.7576960558377633, + "pass@16": 0.7810744308402775, + "pass@32": 0.8012472306707449, + "pass@64": 0.8195398904071645, + "pass@128": 0.8366328916601715 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9239120015337423, + "pass@2": 0.9454351540988355, + "pass@4": 0.9593524645692291, + "pass@8": 0.9691033208480212, + "pass@16": 0.9760508641263232, + "pass@32": 0.9818746126405525, + "pass@64": 0.9878373267292176, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9984564969604863, + "pass@2": 0.9993156606753941, + "pass@4": 0.9998513137035683, + "pass@8": 0.999993739366535, + "pass@16": 0.9999999933961763, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969460227272727, + "pass@2": 0.9988092042471962, + "pass@4": 0.9997825655315814, + "pass@8": 0.9999933566104181, + "pass@16": 0.999999996438375, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9996724013418442, + "pass@4": 0.9999862412723556, + "pass@8": 0.999999975989384, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9799592391304348, + "pass@2": 0.984868909049412, + "pass@4": 0.9885561446123581, + "pass@8": 0.9915921962517327, + "pass@16": 0.993767491284291, + "pass@32": 0.9948652815852641, + "pass@64": 0.9956520789485371, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9882476719197708, + "pass@2": 0.9931144665297928, + "pass@4": 0.9947072929458316, + "pass@8": 0.9954162070318462, + "pass@16": 0.9963557763664397, + "pass@32": 0.9977304914249616, + "pass@64": 0.9992067323962728, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866329479768786, + "pass@2": 0.9905159073323926, + "pass@4": 0.9934451435983797, + "pass@8": 0.99603050505388, + "pass@16": 0.9979728178127817, + "pass@32": 0.9990894429436349, + "pass@64": 0.9998278630547335, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9754401408450705, + "pass@2": 0.9891212432072756, + "pass@4": 0.995878176671578, + "pass@8": 0.9984830423791722, + "pass@16": 0.9995299276902677, + "pass@32": 0.9999462568241625, + "pass@64": 0.9999998252073421, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9667617889221557, + "pass@2": 0.979590400891131, + "pass@4": 0.9860774356798212, + "pass@8": 0.9907092343878937, + "pass@16": 0.9937065287352138, + "pass@32": 0.9952315126113409, + "pass@64": 0.9962629619361241, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.8809299698795181, + "pass@2": 0.9194373458400537, + "pass@4": 0.9411916425544699, + "pass@8": 0.9545487908977287, + "pass@16": 0.9626235409000866, + "pass@32": 0.9688801959881596, + "pass@64": 0.9750893893190955, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.7426283307210031, + "pass@2": 0.8214161195171922, + "pass@4": 0.8708843078588523, + "pass@8": 0.9025153476243613, + "pass@16": 0.9236789138226168, + "pass@32": 0.9398016133683879, + "pass@64": 0.9512363959324338, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.44568452380952384, + "pass@2": 0.5398945995721127, + "pass@4": 0.6209990255944895, + "pass@8": 0.6893619895760472, + "pass@16": 0.7460630135915871, + "pass@32": 0.7918121601901343, + "pass@64": 0.8266485140373896, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.3906935307017544, + "pass@2": 0.4798242736105354, + "pass@4": 0.5533021609689135, + "pass@8": 0.610189362344645, + "pass@16": 0.655899075225643, + "pass@32": 0.6963984026027993, + "pass@64": 0.734174599789174, + "pass@128": 0.7748538011695907 + }, + "15": { + "pass@1": 0.23965435606060606, + "pass@2": 0.32455708661417326, + "pass@4": 0.40661831504016543, + "pass@8": 0.47953448478625676, + "pass@16": 0.5390197751191101, + "pass@32": 0.5891773536829282, + "pass@64": 0.6362165742536966, + "pass@128": 0.6818181818181818 + }, + "16": { + "pass@1": 0.21923967236467237, + "pass@2": 0.28664480842138323, + "pass@4": 0.34611463497618344, + "pass@8": 0.39435134301166086, + "pass@16": 0.4344641369924253, + "pass@32": 0.47232332153629814, + "pass@64": 0.5126976898405665, + "pass@128": 0.5584045584045584 + }, + "17": { + "pass@1": 0.20615780279503104, + "pass@2": 0.28085950872988696, + "pass@4": 0.3537120633600925, + "pass@8": 0.4190615788920885, + "pass@16": 0.4747636461754707, + "pass@32": 0.5222883827191508, + "pass@64": 0.5621582601092799, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.17676930147058822, + "pass@2": 0.24354989289022702, + "pass@4": 0.3090393573965019, + "pass@8": 0.367779967747244, + "pass@16": 0.4192643302519299, + "pass@32": 0.46349945288705857, + "pass@64": 0.5030580827068063, + "pass@128": 0.5411764705882353 + }, + "19": { + "pass@1": 0.15590122767857142, + "pass@2": 0.21318721878515182, + "pass@4": 0.2684598802939811, + "pass@8": 0.31754432950504796, + "pass@16": 0.3607783229282108, + "pass@32": 0.4025381677000241, + "pass@64": 0.44503200389778724, + "pass@128": 0.48214285714285715 + }, + "20": { + "pass@1": 0.14338510479041916, + "pass@2": 0.20299386463293878, + "pass@4": 0.26204798052938, + "pass@8": 0.3151829118182685, + "pass@16": 0.3618771130181657, + "pass@32": 0.40376215264946563, + "pass@64": 0.4417034842332708, + "pass@128": 0.47305389221556887 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 526612, + "answer_accuracy": 0.6573184614155616, + "avg_response_len": 280.08822670354687, + "resp_tokens_sum": 224393243, + "pass_at_k": { + "pass@1": 0.6573184614155616, + "pass@2": 0.7002938209922582, + "pass@4": 0.7359655004180538, + "pass@8": 0.7652439670407067, + "pass@16": 0.7896710217064363, + "pass@32": 0.8103663998207761, + "pass@64": 0.8283027891273101, + "pass@128": 0.8443840869148427 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9116371268656717, + "pass@2": 0.937442340463039, + "pass@4": 0.9559725072612185, + "pass@8": 0.9708405893181536, + "pass@16": 0.9816621823448725, + "pass@32": 0.988235832629917, + "pass@64": 0.9916750917964882, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9968303571428572, + "pass@2": 0.9981032058492688, + "pass@4": 0.9989035175960147, + "pass@8": 0.9995961274270628, + "pass@16": 0.999951050006438, + "pass@32": 0.9999995952745215, + "pass@64": 0.9999999999993562, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9950573979591837, + "pass@2": 0.997648771263286, + "pass@4": 0.9992110229114947, + "pass@8": 0.9998411770221788, + "pass@16": 0.9999932951151359, + "pass@32": 0.9999999952610663, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999314692982456, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9557444852941176, + "pass@2": 0.9666801760074109, + "pass@4": 0.9756720058889695, + "pass@8": 0.9828397062205556, + "pass@16": 0.9880081658678815, + "pass@32": 0.9912545655027337, + "pass@64": 0.9932100684579969, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9848186119873817, + "pass@2": 0.9906298125884897, + "pass@4": 0.9938137757425434, + "pass@8": 0.9955692880639988, + "pass@16": 0.9964674152713848, + "pass@32": 0.9968142019431131, + "pass@64": 0.9968453877933975, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9852650316455697, + "pass@2": 0.9892409735373268, + "pass@4": 0.9916162663211403, + "pass@8": 0.993687297013313, + "pass@16": 0.9952823765021533, + "pass@32": 0.9962915698626211, + "pass@64": 0.9967916970704975, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9691019144144144, + "pass@2": 0.9787938775389561, + "pass@4": 0.9854665815984716, + "pass@8": 0.9905508787963346, + "pass@16": 0.99471446375741, + "pass@32": 0.9981945109364294, + "pass@64": 0.9998532677372303, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.967820538585209, + "pass@2": 0.983230530166848, + "pass@4": 0.9906003204583351, + "pass@8": 0.9945133453287406, + "pass@16": 0.997474489361695, + "pass@32": 0.9994567770978201, + "pass@64": 0.9999874533045885, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8813740079365079, + "pass@2": 0.9245094363204603, + "pass@4": 0.9467500633849341, + "pass@8": 0.9600410828745914, + "pass@16": 0.9696817791361351, + "pass@32": 0.9763972676484557, + "pass@64": 0.9817506994572943, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.7168147935779816, + "pass@2": 0.8032827963591705, + "pass@4": 0.860777312812962, + "pass@8": 0.8982016033603121, + "pass@16": 0.9219491321540287, + "pass@32": 0.9387358226486271, + "pass@64": 0.9511848296368877, + "pass@128": 0.9602446483180428 + }, + "13": { + "pass@1": 0.4994454089506173, + "pass@2": 0.5972381707494899, + "pass@4": 0.6730374565447833, + "pass@8": 0.7285503146535492, + "pass@16": 0.7699852223612504, + "pass@32": 0.8015219546127118, + "pass@64": 0.8283377588720496, + "pass@128": 0.8580246913580247 + }, + "14": { + "pass@1": 0.4094723664688427, + "pass@2": 0.5120147374938671, + "pass@4": 0.5994016718792938, + "pass@8": 0.6680707784308229, + "pass@16": 0.7198891513690838, + "pass@32": 0.7576297630933031, + "pass@64": 0.7875890019240686, + "pass@128": 0.8160237388724035 + }, + "15": { + "pass@1": 0.2975260416666667, + "pass@2": 0.38990275955088916, + "pass@4": 0.4689121339577924, + "pass@8": 0.5342327424552892, + "pass@16": 0.5893543969179779, + "pass@32": 0.6388490866897505, + "pass@64": 0.6837608960192697, + "pass@128": 0.7253086419753086 + }, + "16": { + "pass@1": 0.21773598820058998, + "pass@2": 0.28413271142545243, + "pass@4": 0.3442716314221784, + "pass@8": 0.3984094788613569, + "pass@16": 0.4512509938656513, + "pass@32": 0.5028976983193929, + "pass@64": 0.5488213270348411, + "pass@128": 0.5899705014749262 + }, + "17": { + "pass@1": 0.18728821536144577, + "pass@2": 0.2574441466654016, + "pass@4": 0.3238519578877939, + "pass@8": 0.3830902701560233, + "pass@16": 0.4372752498631445, + "pass@32": 0.4890639818691892, + "pass@64": 0.5392894920468873, + "pass@128": 0.5843373493975904 + }, + "18": { + "pass@1": 0.18455141129032257, + "pass@2": 0.2487045974091947, + "pass@4": 0.3105201652212829, + "pass@8": 0.3670708924370873, + "pass@16": 0.41740293677982304, + "pass@32": 0.46201414725048817, + "pass@64": 0.5028759259860611, + "pass@128": 0.5387096774193548 + }, + "19": { + "pass@1": 0.15445619335347432, + "pass@2": 0.21377112722125746, + "pass@4": 0.2710688189089655, + "pass@8": 0.32411792417505997, + "pass@16": 0.37230826483691687, + "pass@32": 0.41212377291561847, + "pass@64": 0.44557025127759675, + "pass@128": 0.48036253776435045 + }, + "20": { + "pass@1": 0.155640015015015, + "pass@2": 0.21806833900829967, + "pass@4": 0.2809428210550256, + "pass@8": 0.33929357303967483, + "pass@16": 0.39102895548614497, + "pass@32": 0.43822131708853057, + "pass@64": 0.4817266632883948, + "pass@128": 0.5135135135135135 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f4e43cd80bf47dced68d07a6dc3477dbdbbea2f5 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4248179bac7d0e605c613d864ae679a299d6b581eb78d676261c1535946278a +size 1791857453 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..896da2ec10cdad7cab5b954c449cbee98abe1bfe --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9736197916666667, + "avg_response_len": 199.40330902777777, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9736197916666667, + "pass@2": 0.9828890666010458, + "pass@4": 0.9885815226638323, + "pass@8": 0.9924006922485324, + "pass@16": 0.9949174318120703, + "pass@32": 0.9964302425629281, + "pass@64": 0.9973458683385878, + "pass@128": 0.9981111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.91996875, + "pass@2": 0.9440107037401584, + "pass@4": 0.9613539403824525, + "pass@8": 0.9741359237380044, + "pass@16": 0.9829231726689458, + "pass@32": 0.9882494732434092, + "pass@64": 0.9911248980813431, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9965859375, + "pass@2": 0.9978051181102363, + "pass@4": 0.9985503377390326, + "pass@8": 0.9992304553772023, + "pass@16": 0.9997471359244668, + "pass@32": 0.999971111577183, + "pass@64": 0.9999998611804973, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9954296875, + "pass@2": 0.9978746309055119, + "pass@4": 0.9992385676790402, + "pass@8": 0.9998410347048978, + "pass@16": 0.999991556358295, + "pass@32": 0.9999999849666296, + "pass@64": 0.9999999999999996, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974921875, + "pass@2": 0.99872625492126, + "pass@4": 0.9994447872140982, + "pass@8": 0.999863914537571, + "pass@16": 0.9999906599397986, + "pass@32": 0.9999999698373394, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9640390625, + "pass@2": 0.9725996555118112, + "pass@4": 0.978943745781777, + "pass@8": 0.9838275275097776, + "pass@16": 0.9877105076797561, + "pass@32": 0.990995325109703, + "pass@64": 0.9936967414796087, + "pass@128": 0.996 + }, + "6": { + "pass@1": 0.9775390625, + "pass@2": 0.9868101624015748, + "pass@4": 0.9915983475815522, + "pass@8": 0.9938845047599437, + "pass@16": 0.9949757492387002, + "pass@32": 0.9954836866220733, + "pass@64": 0.9959999856206729, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9799453125, + "pass@2": 0.9863713090551185, + "pass@4": 0.9901457638732658, + "pass@8": 0.9929983424627827, + "pass@16": 0.9949317181005914, + "pass@32": 0.9960799208334501, + "pass@64": 0.9969573493204971, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.969875, + "pass@2": 0.9825403543307087, + "pass@4": 0.9894559330708662, + "pass@8": 0.9940618461376957, + "pass@16": 0.9971324044351269, + "pass@32": 0.9986673216314584, + "pass@64": 0.9993766723748042, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.961703125, + "pass@2": 0.9792634104330715, + "pass@4": 0.9885022806524184, + "pass@8": 0.9937626810089238, + "pass@16": 0.9968539819629457, + "pass@32": 0.9984253892451028, + "pass@64": 0.9989573069898666, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.91996875, + "2": 0.9965859375, + "3": 0.9954296875, + "4": 0.9974921875, + "5": 0.9640390625, + "6": 0.9775390625, + "7": 0.9799453125, + "8": 0.969875, + "9": 0.961703125 + }, + "per_op_avg_response_len": { + "10": 278.728375, + "2": 115.734046875, + "3": 144.5596015625, + "4": 159.2734296875, + "5": 174.16815625, + "6": 198.9096953125, + "7": 223.0949453125, + "8": 242.3994921875, + "9": 257.7620390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.728375, + "2": 115.734046875, + "3": 144.5596015625, + "4": 159.2734296875, + "5": 174.16815625, + "6": 198.9096953125, + "7": 223.0949453125, + "8": 242.3994921875, + "9": 257.7620390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 368747, + "answer_accuracy": 0.9729266928402567, + "avg_response_len": 189.78649263340088, + "resp_tokens_sum": 71930599, + "pass_at_k": { + "pass@1": 0.9729266928402567, + "pass@2": 0.9821440205612605, + "pass@4": 0.9875923418640558, + "pass@8": 0.991384693001854, + "pass@16": 0.9940999990363503, + "pass@32": 0.9958890618707124, + "pass@64": 0.9972883530403038, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9248939896755162, + "pass@2": 0.9462058393143337, + "pass@4": 0.9605366295474124, + "pass@8": 0.9713611235954099, + "pass@16": 0.9800066086763779, + "pass@32": 0.9868547832437569, + "pass@64": 0.9923681900521832, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9971524532710281, + "pass@2": 0.997739056712537, + "pass@4": 0.9983754203621744, + "pass@8": 0.9991721938425355, + "pass@16": 0.9998009003384084, + "pass@32": 0.9999920326399729, + "pass@64": 0.9999999987627288, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975391819571865, + "pass@2": 0.9994220905873004, + "pass@4": 0.9999528238442673, + "pass@8": 0.9999996971931873, + "pass@16": 0.9999999999944118, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9949462890625, + "pass@2": 0.9967554287647638, + "pass@4": 0.9983901592379079, + "pass@8": 0.9995789226623526, + "pass@16": 0.9999708149888308, + "pass@32": 0.9999999057416857, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9621031746031746, + "pass@2": 0.9698705630546183, + "pass@4": 0.9753851185268506, + "pass@8": 0.9798042600934502, + "pass@16": 0.9836601066054623, + "pass@32": 0.9872888783520354, + "pass@64": 0.9907961636655847, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9682354041916168, + "pass@2": 0.9816244725116692, + "pass@4": 0.9886694962156675, + "pass@8": 0.992154678669665, + "pass@16": 0.9939489144577033, + "pass@32": 0.994741302662305, + "pass@64": 0.995508975120257, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9770247781065089, + "pass@2": 0.9843062042817871, + "pass@4": 0.9889038042766843, + "pass@8": 0.9927051296156951, + "pass@16": 0.9955222820647343, + "pass@32": 0.9972433727539783, + "pass@64": 0.9984797932474441, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9729817708333334, + "pass@2": 0.9827751962194632, + "pass@4": 0.9881974047114304, + "pass@8": 0.9926713618110146, + "pass@16": 0.9959623220314486, + "pass@32": 0.9974855169110697, + "pass@64": 0.9983969843073067, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9644366197183099, + "pass@2": 0.9820408672507485, + "pass@4": 0.9905973869463501, + "pass@8": 0.9953054121741431, + "pass@16": 0.9980914371290873, + "pass@32": 0.9993323812044371, + "pass@64": 0.9999188136773991, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 380890, + "answer_accuracy": 0.9750010239187418, + "avg_response_len": 197.3905686844692, + "resp_tokens_sum": 77111810, + "pass_at_k": { + "pass@1": 0.9750010239187418, + "pass@2": 0.9843577465660839, + "pass@4": 0.9898513173852935, + "pass@8": 0.9931954755916081, + "pass@16": 0.9952895075696546, + "pass@32": 0.9965716503400766, + "pass@64": 0.9972401532300581, + "pass@128": 0.9977064220183486 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9205329754601227, + "pass@2": 0.9431817152553016, + "pass@4": 0.9587725736736894, + "pass@8": 0.9698210443057249, + "pass@16": 0.9783974902701662, + "pass@32": 0.9842556829652028, + "pass@64": 0.9868527944754925, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9962481003039514, + "pass@2": 0.9974010063901586, + "pass@4": 0.9979578029570011, + "pass@8": 0.9986493669171034, + "pass@16": 0.9994341559274335, + "pass@32": 0.9999199744887792, + "pass@64": 0.9999995792634084, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9943418560606061, + "pass@2": 0.996974916487712, + "pass@4": 0.9986633554328435, + "pass@8": 0.9996870700423274, + "pass@16": 0.9999813836429393, + "pass@32": 0.999999959369951, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9977579511834319, + "pass@2": 0.9993528106508875, + "pass@4": 0.9998821282694692, + "pass@8": 0.9999960334397089, + "pass@16": 0.9999999974655996, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9718523550724638, + "pass@2": 0.9773992924797448, + "pass@4": 0.9825246996299374, + "pass@8": 0.9874108605762586, + "pass@16": 0.9912208543323465, + "pass@32": 0.9941367591421882, + "pass@64": 0.9967430652968408, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9824050859598854, + "pass@2": 0.9914463145545203, + "pass@4": 0.9956285275407906, + "pass@8": 0.9968915638760587, + "pass@16": 0.9971227705571397, + "pass@32": 0.9971346536923668, + "pass@64": 0.9971346704871058, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9837653540462428, + "pass@2": 0.9907264120886624, + "pass@4": 0.9944128431742274, + "pass@8": 0.9963867706256992, + "pass@16": 0.9970495251707892, + "pass@32": 0.9971095689707938, + "pass@64": 0.9971098265895346, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9706866197183098, + "pass@2": 0.987252550737496, + "pass@4": 0.9947429746457749, + "pass@8": 0.9977459292370181, + "pass@16": 0.999301853323875, + "pass@32": 0.9999206896276759, + "pass@64": 0.9999996096160376, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9561190119760479, + "pass@2": 0.9741652276156351, + "pass@4": 0.9849497023949854, + "pass@8": 0.9913124752168228, + "pass@16": 0.9944909189241692, + "pass@32": 0.9962477073890863, + "pass@64": 0.9969666351508508, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 371973, + "answer_accuracy": 0.9728955682959491, + "avg_response_len": 210.99295645714764, + "resp_tokens_sum": 80670203, + "pass_at_k": { + "pass@1": 0.9728955682959491, + "pass@2": 0.9821269876156261, + "pass@4": 0.9882646665735247, + "pass@8": 0.992595769250994, + "pass@16": 0.995347576852833, + "pass@32": 0.9968222276629579, + "pass@64": 0.9975108985727513, + "pass@128": 0.9979912956143288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9144356343283582, + "pass@2": 0.9425960747443888, + "pass@4": 0.9646930267112135, + "pass@8": 0.9811428131208776, + "pass@16": 0.9902786582077016, + "pass@32": 0.9935473106779685, + "pass@64": 0.9940240915063959, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9963839285714285, + "pass@2": 0.9982455708661419, + "pass@4": 0.9992677446569179, + "pass@8": 0.9998301126800612, + "pass@16": 0.9999920274734635, + "pass@32": 0.9999999928369812, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9944651967930029, + "pass@2": 0.9972649703863546, + "pass@4": 0.9991110407846251, + "pass@8": 0.9998379026494393, + "pass@16": 0.9999932937549632, + "pass@32": 0.9999999952610663, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999510751945482, + "pass@4": 0.9999993476692606, + "pass@8": 0.9999999999502821, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9579044117647059, + "pass@2": 0.9702578595414542, + "pass@4": 0.9786070943337964, + "pass@8": 0.9839189372986221, + "pass@16": 0.9879011157481662, + "pass@32": 0.991241607425756, + "pass@64": 0.9932929188162924, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9819844242902208, + "pass@2": 0.9871697943317023, + "pass@4": 0.9902472546807043, + "pass@8": 0.9923964867240731, + "pass@16": 0.9936938955374945, + "pass@32": 0.994448256764124, + "pass@64": 0.9952681007271514, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9788864715189873, + "pass@2": 0.983811624763281, + "pass@4": 0.9868020072451386, + "pass@8": 0.9896018544816012, + "pass@16": 0.9919811742203117, + "pass@32": 0.9937080695528824, + "pass@64": 0.9951619595027914, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9660989114114115, + "pass@2": 0.9772968096048807, + "pass@4": 0.984998759164113, + "pass@8": 0.9914371662868396, + "pass@16": 0.9959159220160351, + "pass@32": 0.9984384250371764, + "pass@64": 0.999630486057751, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9645799839228296, + "pass@2": 0.9815682203711674, + "pass@4": 0.9899260697316372, + "pass@8": 0.9946330963494676, + "pass@16": 0.9979792760818236, + "pass@32": 0.9997288091626138, + "pass@64": 0.9999976591447138, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4432078125, + "avg_response_len": 340.78941953125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4432078125, + "pass@2": 0.5174025590551129, + "pass@4": 0.5798022140607307, + "pass@8": 0.6315635445530711, + "pass@16": 0.6750730398377451, + "pass@32": 0.7128041221508825, + "pass@64": 0.7462023426101437, + "pass@128": 0.7759 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9032265625, + "pass@2": 0.9364288877952763, + "pass@4": 0.9558144556617925, + "pass@8": 0.9676226180268027, + "pass@16": 0.9751345363769143, + "pass@32": 0.980559591503808, + "pass@64": 0.9854267407965909, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.835890625, + "pass@2": 0.8854387303149611, + "pass@4": 0.9128389102924639, + "pass@8": 0.9303765088015541, + "pass@16": 0.9430146794272325, + "pass@32": 0.9521916505514397, + "pass@64": 0.9598103872142445, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.6519296875, + "pass@2": 0.7413225885826774, + "pass@4": 0.8036862839332581, + "pass@8": 0.8467542446993395, + "pass@16": 0.8762871486525114, + "pass@32": 0.8980258735531979, + "pass@64": 0.9166463553601779, + "pass@128": 0.933 + }, + "14": { + "pass@1": 0.5386640625, + "pass@2": 0.6328204970472447, + "pass@4": 0.7064343655793021, + "pass@8": 0.7630503582736895, + "pass@16": 0.807217143852765, + "pass@32": 0.8419916334661619, + "pass@64": 0.8697842799666274, + "pass@128": 0.89 + }, + "15": { + "pass@1": 0.420640625, + "pass@2": 0.5211077755905517, + "pass@4": 0.60270849671916, + "pass@8": 0.6671531220031565, + "pass@16": 0.7201534841774442, + "pass@32": 0.7648040005237096, + "pass@64": 0.8003557337258144, + "pass@128": 0.828 + }, + "16": { + "pass@1": 0.287453125, + "pass@2": 0.3752082923228351, + "pass@4": 0.45372917491563564, + "pass@8": 0.5201479130236136, + "pass@16": 0.5770370983307045, + "pass@32": 0.6284656751544069, + "pass@64": 0.6767553239592762, + "pass@128": 0.724 + }, + "17": { + "pass@1": 0.2477109375, + "pass@2": 0.32817642716535456, + "pass@4": 0.40190449146981594, + "pass@8": 0.46670493088482984, + "pass@16": 0.5251041397298073, + "pass@32": 0.5815003030233882, + "pass@64": 0.6365153980234262, + "pass@128": 0.687 + }, + "18": { + "pass@1": 0.2187421875, + "pass@2": 0.2972192421259842, + "pass@4": 0.37341992894638143, + "pass@8": 0.4417571057508958, + "pass@16": 0.5007237224326827, + "pass@32": 0.5517062692844494, + "pass@64": 0.5969404968407211, + "pass@128": 0.64 + }, + "19": { + "pass@1": 0.165921875, + "pass@2": 0.22998216043307082, + "pass@4": 0.29479702277840264, + "pass@8": 0.3557075550684329, + "pass@16": 0.4107528754096038, + "pass@32": 0.460850124712799, + "pass@64": 0.5077246129223387, + "pass@128": 0.553 + }, + "20": { + "pass@1": 0.1618984375, + "pass@2": 0.22632098917322818, + "pass@4": 0.29268901031121103, + "pass@8": 0.3563610889983621, + "pass@16": 0.41530556998766033, + "pass@32": 0.4679460997354523, + "pass@64": 0.5120640972921665, + "pass@128": 0.547 + } + }, + "per_op_accuracy": { + "11": 0.9032265625, + "12": 0.835890625, + "13": 0.6519296875, + "14": 0.5386640625, + "15": 0.420640625, + "16": 0.287453125, + "17": 0.2477109375, + "18": 0.2187421875, + "19": 0.165921875, + "20": 0.1618984375 + }, + "per_op_avg_response_len": { + "11": 296.1493984375, + "12": 306.707375, + "13": 321.816875, + "14": 340.003796875, + "15": 348.7493359375, + "16": 352.550421875, + "17": 361.423234375, + "18": 360.763625, + "19": 357.3403828125, + "20": 362.38975 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.1493984375, + "12": 306.707375, + "13": 321.816875, + "14": 340.003796875, + "15": 348.7493359375, + "16": 352.550421875, + "17": 361.423234375, + "18": 360.763625, + "19": 357.3403828125, + "20": 362.38975 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 191628, + "answer_accuracy": 0.44490156017830607, + "avg_response_len": 323.4324317421991, + "resp_tokens_sum": 139308817, + "pass_at_k": { + "pass@1": 0.44490156017830607, + "pass@2": 0.5163603371318961, + "pass@4": 0.575135991795604, + "pass@8": 0.6232565625893707, + "pass@16": 0.6639309549458606, + "pass@32": 0.6994803934575693, + "pass@64": 0.7310869408893447, + "pass@128": 0.7598811292719168 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8889429886685553, + "pass@2": 0.9242913664651687, + "pass@4": 0.9465331700747036, + "pass@8": 0.9604262141604022, + "pass@16": 0.9689611062295932, + "pass@32": 0.9749640416956897, + "pass@64": 0.9803316295352276, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8323181497175142, + "pass@2": 0.8870070399039109, + "pass@4": 0.9188809555585208, + "pass@8": 0.9380959870647928, + "pass@16": 0.9504149057699475, + "pass@32": 0.9579377264267592, + "pass@64": 0.9628166202895165, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.6575480015673981, + "pass@2": 0.7480878051489646, + "pass@4": 0.8102339428378657, + "pass@8": 0.8522237777096143, + "pass@16": 0.882257665109769, + "pass@32": 0.9087626399289626, + "pass@64": 0.93377578450576, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.5357768691588785, + "pass@2": 0.6224040737606394, + "pass@4": 0.6857702331601073, + "pass@8": 0.7324715141602004, + "pass@16": 0.7720395421287615, + "pass@32": 0.8098585396648957, + "pass@64": 0.8454894274337638, + "pass@128": 0.8691588785046729 + }, + "15": { + "pass@1": 0.4288972182080925, + "pass@2": 0.5222473658458879, + "pass@4": 0.5974337641501454, + "pass@8": 0.6592894122612182, + "pass@16": 0.7139905030261611, + "pass@32": 0.76220652456853, + "pass@64": 0.8015069468721714, + "pass@128": 0.8323699421965318 + }, + "16": { + "pass@1": 0.27419354838709675, + "pass@2": 0.3570941548133095, + "pass@4": 0.4288660543440137, + "pass@8": 0.48969756116313695, + "pass@16": 0.5435783504560074, + "pass@32": 0.5964149406645595, + "pass@64": 0.6502359398435504, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.24882586705202311, + "pass@2": 0.32303341620772813, + "pass@4": 0.3890596267590826, + "pass@8": 0.4457163994015934, + "pass@16": 0.4957672003720015, + "pass@32": 0.5433302498892872, + "pass@64": 0.5907082354561285, + "pass@128": 0.638728323699422 + }, + "18": { + "pass@1": 0.21283482142857144, + "pass@2": 0.28537085208098983, + "pass@4": 0.3563121484814401, + "pass@8": 0.42147138400766687, + "pass@16": 0.4769469684933736, + "pass@32": 0.5212049806482858, + "pass@64": 0.5583199932477526, + "pass@128": 0.5942857142857143 + }, + "19": { + "pass@1": 0.16884853603603603, + "pass@2": 0.23270058936889648, + "pass@4": 0.29553178206327807, + "pass@8": 0.35391545800133395, + "pass@16": 0.4071753895217641, + "pass@32": 0.45166801653626926, + "pass@64": 0.48833945361368375, + "pass@128": 0.5285285285285285 + }, + "20": { + "pass@1": 0.17027965465465467, + "pass@2": 0.23516528339362977, + "pass@4": 0.3000875284733552, + "pass@8": 0.3593155322698133, + "pass@16": 0.4107097007726869, + "pass@32": 0.45401753885710516, + "pass@64": 0.4883729600405135, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 188271, + "answer_accuracy": 0.4373675847457627, + "avg_response_len": 335.83635100728515, + "resp_tokens_sum": 144565459, + "pass_at_k": { + "pass@1": 0.4373675847457627, + "pass@2": 0.5119158363595512, + "pass@4": 0.5762741240195127, + "pass@8": 0.6308432449388135, + "pass@16": 0.6764574883535417, + "pass@32": 0.7150494150444255, + "pass@64": 0.7483372014713178, + "pass@128": 0.7781742491822777 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9085325677710844, + "pass@2": 0.9393202732188596, + "pass@4": 0.9578793574184248, + "pass@8": 0.9694906501998141, + "pass@16": 0.9766675676967038, + "pass@32": 0.9821868207145709, + "pass@64": 0.9872097467394703, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8305005877742947, + "pass@2": 0.8724626200232023, + "pass@4": 0.8967490576217158, + "pass@8": 0.914954573017507, + "pass@16": 0.9291971281957504, + "pass@32": 0.9389488700907844, + "pass@64": 0.9468632868540318, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6433385854341737, + "pass@2": 0.7310224778887932, + "pass@4": 0.7918085777618122, + "pass@8": 0.8355008886669074, + "pass@16": 0.8664531704460665, + "pass@32": 0.8875200988088843, + "pass@64": 0.9043131635172054, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5319581505847953, + "pass@2": 0.6288834064327484, + "pass@4": 0.7076227626151991, + "pass@8": 0.7681552997502585, + "pass@16": 0.8127676135435525, + "pass@32": 0.8456231168387941, + "pass@64": 0.868671287789918, + "pass@128": 0.8859649122807017 + }, + "15": { + "pass@1": 0.403811553030303, + "pass@2": 0.5122766791935102, + "pass@4": 0.6013293119610047, + "pass@8": 0.6697900902768419, + "pass@16": 0.7231940412743582, + "pass@32": 0.7662809711021994, + "pass@64": 0.80072356907216, + "pass@128": 0.8272727272727273 + }, + "16": { + "pass@1": 0.27966969373219375, + "pass@2": 0.3633873690468179, + "pass@4": 0.4406781112830982, + "pass@8": 0.5073770521645653, + "pass@16": 0.5641668062665472, + "pass@32": 0.613915678888368, + "pass@64": 0.6607560737801916, + "pass@128": 0.7122507122507122 + }, + "17": { + "pass@1": 0.25317837732919257, + "pass@2": 0.3383595392967182, + "pass@4": 0.4201410638887531, + "pass@8": 0.49519154029212514, + "pass@16": 0.5608408710318087, + "pass@32": 0.620854047451868, + "pass@64": 0.6752767385641311, + "pass@128": 0.7204968944099379 + }, + "18": { + "pass@1": 0.22141544117647058, + "pass@2": 0.30369311602593757, + "pass@4": 0.38339679442643204, + "pass@8": 0.45511805576024483, + "pass@16": 0.5201182357236519, + "pass@32": 0.5796778115707856, + "pass@64": 0.6310170644117857, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.15997023809523808, + "pass@2": 0.22198690007499064, + "pass@4": 0.28720189496402243, + "pass@8": 0.3504679565583006, + "pass@16": 0.4073951120645691, + "pass@32": 0.45981662672399704, + "pass@64": 0.5113304653878958, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.15187593562874252, + "pass@2": 0.21543293825734353, + "pass@4": 0.2814116064833215, + "pass@8": 0.34650411636039585, + "pass@16": 0.4071150011676281, + "pass@32": 0.4586578382580935, + "pass@64": 0.4999206483403107, + "pass@128": 0.5329341317365269 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 187407, + "answer_accuracy": 0.4474685780867971, + "avg_response_len": 363.7305666450183, + "resp_tokens_sum": 152336181, + "pass_at_k": { + "pass@1": 0.4474685780867971, + "pass@2": 0.5241137220125921, + "pass@4": 0.5882272766313009, + "pass@8": 0.640846968119873, + "pass@16": 0.6851088635854434, + "pass@32": 0.7241988125701884, + "pass@64": 0.7595531361432362, + "pass@128": 0.7900366748166259 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.913640873015873, + "pass@2": 0.9469831896012993, + "pass@4": 0.9640390475000145, + "pass@8": 0.9737183128947382, + "pass@16": 0.980436931436705, + "pass@32": 0.9851151120888639, + "pass@64": 0.989257268740163, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.84501624617737, + "pass@2": 0.8963995792097086, + "pass@4": 0.9219941976862989, + "pass@8": 0.9370642831438329, + "pass@16": 0.9484829323860189, + "pass@32": 0.9588899261082772, + "pass@64": 0.9691862847869092, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6558641975308642, + "pass@2": 0.7460109695489451, + "pass@4": 0.810327141688307, + "pass@8": 0.853768649246564, + "pass@16": 0.8812443568927503, + "pass@32": 0.8990306053737257, + "pass@64": 0.9133706195283898, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.548219584569733, + "pass@2": 0.6467378764690762, + "pass@4": 0.7249113825534416, + "pass@8": 0.7869967053818288, + "pass@16": 0.8350917715060479, + "pass@32": 0.8689137574921737, + "pass@64": 0.894055173104561, + "pass@128": 0.913946587537092 + }, + "15": { + "pass@1": 0.42896412037037035, + "pass@2": 0.5288854409691839, + "pass@4": 0.609746115358265, + "pass@8": 0.672864986325362, + "pass@16": 0.7236380756477004, + "pass@32": 0.7660735264175047, + "pass@64": 0.7987517046735475, + "pass@128": 0.8240740740740741 + }, + "16": { + "pass@1": 0.3076373525073746, + "pass@2": 0.4040122058857685, + "pass@4": 0.4899784100549376, + "pass@8": 0.5612162942574591, + "pass@16": 0.6209594710613111, + "pass@32": 0.6728396467804595, + "pass@64": 0.7175716835130616, + "pass@128": 0.7551622418879056 + }, + "17": { + "pass@1": 0.24124623493975902, + "pass@2": 0.32365992019258133, + "pass@4": 0.39760372903311775, + "pass@8": 0.4609499419211275, + "pass@16": 0.5210178552073905, + "pass@32": 0.5831115460307031, + "pass@64": 0.6466603576444395, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.2224798387096774, + "pass@2": 0.30349607886715774, + "pass@4": 0.381792796364164, + "pass@8": 0.45000639480557797, + "pass@16": 0.5062970429482582, + "pass@32": 0.5554647423338157, + "pass@64": 0.6031699913032258, + "pass@128": 0.6483870967741936 + }, + "19": { + "pass@1": 0.16901907099697885, + "pass@2": 0.23536334062849376, + "pass@4": 0.30176768170217383, + "pass@8": 0.36282922704048376, + "pass@16": 0.41776046237208814, + "pass@32": 0.4711368236463992, + "pass@64": 0.5235665815669148, + "pass@128": 0.5709969788519638 + }, + "20": { + "pass@1": 0.16356981981981983, + "pass@2": 0.2283974427183088, + "pass@4": 0.2966017620545179, + "pass@8": 0.3632932188833103, + "pass@16": 0.42811660432542914, + "pass@32": 0.49119081476826765, + "pass@64": 0.5479351503093437, + "pass@128": 0.5915915915915916 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6944555921052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6944555921052632, + "pass@2": 0.7378961678926435, + "pass@4": 0.7734345181358707, + "pass@8": 0.8024864039877715, + "pass@16": 0.8265782781413785, + "pass@32": 0.8471533370829175, + "pass@64": 0.8651650653236149, + "pass@128": 0.8811578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.91996875, + "pass@2": 0.9440107037401584, + "pass@4": 0.9613539403824525, + "pass@8": 0.9741359237380044, + "pass@16": 0.9829231726689458, + "pass@32": 0.9882494732434092, + "pass@64": 0.9911248980813431, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9965859375, + "pass@2": 0.9978051181102363, + "pass@4": 0.9985503377390326, + "pass@8": 0.9992304553772023, + "pass@16": 0.9997471359244668, + "pass@32": 0.999971111577183, + "pass@64": 0.9999998611804973, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9954296875, + "pass@2": 0.9978746309055119, + "pass@4": 0.9992385676790402, + "pass@8": 0.9998410347048978, + "pass@16": 0.999991556358295, + "pass@32": 0.9999999849666296, + "pass@64": 0.9999999999999996, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974921875, + "pass@2": 0.99872625492126, + "pass@4": 0.9994447872140982, + "pass@8": 0.999863914537571, + "pass@16": 0.9999906599397986, + "pass@32": 0.9999999698373394, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9640390625, + "pass@2": 0.9725996555118112, + "pass@4": 0.978943745781777, + "pass@8": 0.9838275275097776, + "pass@16": 0.9877105076797561, + "pass@32": 0.990995325109703, + "pass@64": 0.9936967414796087, + "pass@128": 0.996 + }, + "6": { + "pass@1": 0.9775390625, + "pass@2": 0.9868101624015748, + "pass@4": 0.9915983475815522, + "pass@8": 0.9938845047599437, + "pass@16": 0.9949757492387002, + "pass@32": 0.9954836866220733, + "pass@64": 0.9959999856206729, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9799453125, + "pass@2": 0.9863713090551185, + "pass@4": 0.9901457638732658, + "pass@8": 0.9929983424627827, + "pass@16": 0.9949317181005914, + "pass@32": 0.9960799208334501, + "pass@64": 0.9969573493204971, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.969875, + "pass@2": 0.9825403543307087, + "pass@4": 0.9894559330708662, + "pass@8": 0.9940618461376957, + "pass@16": 0.9971324044351269, + "pass@32": 0.9986673216314584, + "pass@64": 0.9993766723748042, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.961703125, + "pass@2": 0.9792634104330715, + "pass@4": 0.9885022806524184, + "pass@8": 0.9937626810089238, + "pass@16": 0.9968539819629457, + "pass@32": 0.9984253892451028, + "pass@64": 0.9989573069898666, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9032265625, + "pass@2": 0.9364288877952763, + "pass@4": 0.9558144556617925, + "pass@8": 0.9676226180268027, + "pass@16": 0.9751345363769143, + "pass@32": 0.980559591503808, + "pass@64": 0.9854267407965909, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.835890625, + "pass@2": 0.8854387303149611, + "pass@4": 0.9128389102924639, + "pass@8": 0.9303765088015541, + "pass@16": 0.9430146794272325, + "pass@32": 0.9521916505514397, + "pass@64": 0.9598103872142445, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.6519296875, + "pass@2": 0.7413225885826774, + "pass@4": 0.8036862839332581, + "pass@8": 0.8467542446993395, + "pass@16": 0.8762871486525114, + "pass@32": 0.8980258735531979, + "pass@64": 0.9166463553601779, + "pass@128": 0.933 + }, + "14": { + "pass@1": 0.5386640625, + "pass@2": 0.6328204970472447, + "pass@4": 0.7064343655793021, + "pass@8": 0.7630503582736895, + "pass@16": 0.807217143852765, + "pass@32": 0.8419916334661619, + "pass@64": 0.8697842799666274, + "pass@128": 0.89 + }, + "15": { + "pass@1": 0.420640625, + "pass@2": 0.5211077755905517, + "pass@4": 0.60270849671916, + "pass@8": 0.6671531220031565, + "pass@16": 0.7201534841774442, + "pass@32": 0.7648040005237096, + "pass@64": 0.8003557337258144, + "pass@128": 0.828 + }, + "16": { + "pass@1": 0.287453125, + "pass@2": 0.3752082923228351, + "pass@4": 0.45372917491563564, + "pass@8": 0.5201479130236136, + "pass@16": 0.5770370983307045, + "pass@32": 0.6284656751544069, + "pass@64": 0.6767553239592762, + "pass@128": 0.724 + }, + "17": { + "pass@1": 0.2477109375, + "pass@2": 0.32817642716535456, + "pass@4": 0.40190449146981594, + "pass@8": 0.46670493088482984, + "pass@16": 0.5251041397298073, + "pass@32": 0.5815003030233882, + "pass@64": 0.6365153980234262, + "pass@128": 0.687 + }, + "18": { + "pass@1": 0.2187421875, + "pass@2": 0.2972192421259842, + "pass@4": 0.37341992894638143, + "pass@8": 0.4417571057508958, + "pass@16": 0.5007237224326827, + "pass@32": 0.5517062692844494, + "pass@64": 0.5969404968407211, + "pass@128": 0.64 + }, + "19": { + "pass@1": 0.165921875, + "pass@2": 0.22998216043307082, + "pass@4": 0.29479702277840264, + "pass@8": 0.3557075550684329, + "pass@16": 0.4107528754096038, + "pass@32": 0.460850124712799, + "pass@64": 0.5077246129223387, + "pass@128": 0.553 + }, + "20": { + "pass@1": 0.1618984375, + "pass@2": 0.22632098917322818, + "pass@4": 0.29268901031121103, + "pass@8": 0.3563610889983621, + "pass@16": 0.41530556998766033, + "pass@32": 0.4679460997354523, + "pass@64": 0.5120640972921665, + "pass@128": 0.547 + } + }, + "per_op_accuracy": { + "10": 0.91996875, + "2": 0.9965859375, + "3": 0.9954296875, + "4": 0.9974921875, + "5": 0.9640390625, + "6": 0.9775390625, + "7": 0.9799453125, + "8": 0.969875, + "9": 0.961703125, + "11": 0.9032265625, + "12": 0.835890625, + "13": 0.6519296875, + "14": 0.5386640625, + "15": 0.420640625, + "16": 0.287453125, + "17": 0.2477109375, + "18": 0.2187421875, + "19": 0.165921875, + "20": 0.1618984375 + }, + "per_op_avg_response_len": { + "10": 278.728375, + "2": 115.734046875, + "3": 144.5596015625, + "4": 159.2734296875, + "5": 174.16815625, + "6": 198.9096953125, + "7": 223.0949453125, + "8": 242.3994921875, + "9": 257.7620390625, + "11": 296.1493984375, + "12": 306.707375, + "13": 321.816875, + "14": 340.003796875, + "15": 348.7493359375, + "16": 352.550421875, + "17": 361.423234375, + "18": 360.763625, + "19": 357.3403828125, + "20": 362.38975 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.728375, + "2": 115.734046875, + "3": 144.5596015625, + "4": 159.2734296875, + "5": 174.16815625, + "6": 198.9096953125, + "7": 223.0949453125, + "8": 242.3994921875, + "9": 257.7620390625, + "11": 296.1493984375, + "12": 306.707375, + "13": 321.816875, + "14": 340.003796875, + "15": 348.7493359375, + "16": 352.550421875, + "17": 361.423234375, + "18": 360.763625, + "19": 357.3403828125, + "20": 362.38975 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 560375, + "answer_accuracy": 0.6920533808883971, + "avg_response_len": 260.877005611761, + "resp_tokens_sum": 211239416, + "pass_at_k": { + "pass@1": 0.6920533808883971, + "pass@2": 0.7343789091575517, + "pass@4": 0.7681937301061701, + "pass@8": 0.795565666944629, + "pass@16": 0.8184726146916678, + "pass@32": 0.8382198919038769, + "pass@64": 0.8556873805635508, + "pass@128": 0.8716408472968701 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9248939896755162, + "pass@2": 0.9462058393143337, + "pass@4": 0.9605366295474124, + "pass@8": 0.9713611235954099, + "pass@16": 0.9800066086763779, + "pass@32": 0.9868547832437569, + "pass@64": 0.9923681900521832, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9971524532710281, + "pass@2": 0.997739056712537, + "pass@4": 0.9983754203621744, + "pass@8": 0.9991721938425355, + "pass@16": 0.9998009003384084, + "pass@32": 0.9999920326399729, + "pass@64": 0.9999999987627288, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975391819571865, + "pass@2": 0.9994220905873004, + "pass@4": 0.9999528238442673, + "pass@8": 0.9999996971931873, + "pass@16": 0.9999999999944118, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9949462890625, + "pass@2": 0.9967554287647638, + "pass@4": 0.9983901592379079, + "pass@8": 0.9995789226623526, + "pass@16": 0.9999708149888308, + "pass@32": 0.9999999057416857, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9621031746031746, + "pass@2": 0.9698705630546183, + "pass@4": 0.9753851185268506, + "pass@8": 0.9798042600934502, + "pass@16": 0.9836601066054623, + "pass@32": 0.9872888783520354, + "pass@64": 0.9907961636655847, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9682354041916168, + "pass@2": 0.9816244725116692, + "pass@4": 0.9886694962156675, + "pass@8": 0.992154678669665, + "pass@16": 0.9939489144577033, + "pass@32": 0.994741302662305, + "pass@64": 0.995508975120257, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9770247781065089, + "pass@2": 0.9843062042817871, + "pass@4": 0.9889038042766843, + "pass@8": 0.9927051296156951, + "pass@16": 0.9955222820647343, + "pass@32": 0.9972433727539783, + "pass@64": 0.9984797932474441, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9729817708333334, + "pass@2": 0.9827751962194632, + "pass@4": 0.9881974047114304, + "pass@8": 0.9926713618110146, + "pass@16": 0.9959623220314486, + "pass@32": 0.9974855169110697, + "pass@64": 0.9983969843073067, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9644366197183099, + "pass@2": 0.9820408672507485, + "pass@4": 0.9905973869463501, + "pass@8": 0.9953054121741431, + "pass@16": 0.9980914371290873, + "pass@32": 0.9993323812044371, + "pass@64": 0.9999188136773991, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8889429886685553, + "pass@2": 0.9242913664651687, + "pass@4": 0.9465331700747036, + "pass@8": 0.9604262141604022, + "pass@16": 0.9689611062295932, + "pass@32": 0.9749640416956897, + "pass@64": 0.9803316295352276, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8323181497175142, + "pass@2": 0.8870070399039109, + "pass@4": 0.9188809555585208, + "pass@8": 0.9380959870647928, + "pass@16": 0.9504149057699475, + "pass@32": 0.9579377264267592, + "pass@64": 0.9628166202895165, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.6575480015673981, + "pass@2": 0.7480878051489646, + "pass@4": 0.8102339428378657, + "pass@8": 0.8522237777096143, + "pass@16": 0.882257665109769, + "pass@32": 0.9087626399289626, + "pass@64": 0.93377578450576, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.5357768691588785, + "pass@2": 0.6224040737606394, + "pass@4": 0.6857702331601073, + "pass@8": 0.7324715141602004, + "pass@16": 0.7720395421287615, + "pass@32": 0.8098585396648957, + "pass@64": 0.8454894274337638, + "pass@128": 0.8691588785046729 + }, + "15": { + "pass@1": 0.4288972182080925, + "pass@2": 0.5222473658458879, + "pass@4": 0.5974337641501454, + "pass@8": 0.6592894122612182, + "pass@16": 0.7139905030261611, + "pass@32": 0.76220652456853, + "pass@64": 0.8015069468721714, + "pass@128": 0.8323699421965318 + }, + "16": { + "pass@1": 0.27419354838709675, + "pass@2": 0.3570941548133095, + "pass@4": 0.4288660543440137, + "pass@8": 0.48969756116313695, + "pass@16": 0.5435783504560074, + "pass@32": 0.5964149406645595, + "pass@64": 0.6502359398435504, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.24882586705202311, + "pass@2": 0.32303341620772813, + "pass@4": 0.3890596267590826, + "pass@8": 0.4457163994015934, + "pass@16": 0.4957672003720015, + "pass@32": 0.5433302498892872, + "pass@64": 0.5907082354561285, + "pass@128": 0.638728323699422 + }, + "18": { + "pass@1": 0.21283482142857144, + "pass@2": 0.28537085208098983, + "pass@4": 0.3563121484814401, + "pass@8": 0.42147138400766687, + "pass@16": 0.4769469684933736, + "pass@32": 0.5212049806482858, + "pass@64": 0.5583199932477526, + "pass@128": 0.5942857142857143 + }, + "19": { + "pass@1": 0.16884853603603603, + "pass@2": 0.23270058936889648, + "pass@4": 0.29553178206327807, + "pass@8": 0.35391545800133395, + "pass@16": 0.4071753895217641, + "pass@32": 0.45166801653626926, + "pass@64": 0.48833945361368375, + "pass@128": 0.5285285285285285 + }, + "20": { + "pass@1": 0.17027965465465467, + "pass@2": 0.23516528339362977, + "pass@4": 0.3000875284733552, + "pass@8": 0.3593155322698133, + "pass@16": 0.4107097007726869, + "pass@32": 0.45401753885710516, + "pass@64": 0.4883729600405135, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 569161, + "answer_accuracy": 0.69315203624318, + "avg_response_len": 269.96939424201094, + "resp_tokens_sum": 221677269, + "pass_at_k": { + "pass@1": 0.69315203624318, + "pass@2": 0.7366847701008297, + "pass@4": 0.7730375837470743, + "pass@8": 0.8032359196001282, + "pass@16": 0.828144989935398, + "pass@32": 0.84898641615469, + "pass@64": 0.8667552542800026, + "pass@128": 0.8826188620420888 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9205329754601227, + "pass@2": 0.9431817152553016, + "pass@4": 0.9587725736736894, + "pass@8": 0.9698210443057249, + "pass@16": 0.9783974902701662, + "pass@32": 0.9842556829652028, + "pass@64": 0.9868527944754925, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9962481003039514, + "pass@2": 0.9974010063901586, + "pass@4": 0.9979578029570011, + "pass@8": 0.9986493669171034, + "pass@16": 0.9994341559274335, + "pass@32": 0.9999199744887792, + "pass@64": 0.9999995792634084, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9943418560606061, + "pass@2": 0.996974916487712, + "pass@4": 0.9986633554328435, + "pass@8": 0.9996870700423274, + "pass@16": 0.9999813836429393, + "pass@32": 0.999999959369951, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9977579511834319, + "pass@2": 0.9993528106508875, + "pass@4": 0.9998821282694692, + "pass@8": 0.9999960334397089, + "pass@16": 0.9999999974655996, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9718523550724638, + "pass@2": 0.9773992924797448, + "pass@4": 0.9825246996299374, + "pass@8": 0.9874108605762586, + "pass@16": 0.9912208543323465, + "pass@32": 0.9941367591421882, + "pass@64": 0.9967430652968408, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9824050859598854, + "pass@2": 0.9914463145545203, + "pass@4": 0.9956285275407906, + "pass@8": 0.9968915638760587, + "pass@16": 0.9971227705571397, + "pass@32": 0.9971346536923668, + "pass@64": 0.9971346704871058, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9837653540462428, + "pass@2": 0.9907264120886624, + "pass@4": 0.9944128431742274, + "pass@8": 0.9963867706256992, + "pass@16": 0.9970495251707892, + "pass@32": 0.9971095689707938, + "pass@64": 0.9971098265895346, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9706866197183098, + "pass@2": 0.987252550737496, + "pass@4": 0.9947429746457749, + "pass@8": 0.9977459292370181, + "pass@16": 0.999301853323875, + "pass@32": 0.9999206896276759, + "pass@64": 0.9999996096160376, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9561190119760479, + "pass@2": 0.9741652276156351, + "pass@4": 0.9849497023949854, + "pass@8": 0.9913124752168228, + "pass@16": 0.9944909189241692, + "pass@32": 0.9962477073890863, + "pass@64": 0.9969666351508508, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9085325677710844, + "pass@2": 0.9393202732188596, + "pass@4": 0.9578793574184248, + "pass@8": 0.9694906501998141, + "pass@16": 0.9766675676967038, + "pass@32": 0.9821868207145709, + "pass@64": 0.9872097467394703, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8305005877742947, + "pass@2": 0.8724626200232023, + "pass@4": 0.8967490576217158, + "pass@8": 0.914954573017507, + "pass@16": 0.9291971281957504, + "pass@32": 0.9389488700907844, + "pass@64": 0.9468632868540318, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6433385854341737, + "pass@2": 0.7310224778887932, + "pass@4": 0.7918085777618122, + "pass@8": 0.8355008886669074, + "pass@16": 0.8664531704460665, + "pass@32": 0.8875200988088843, + "pass@64": 0.9043131635172054, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5319581505847953, + "pass@2": 0.6288834064327484, + "pass@4": 0.7076227626151991, + "pass@8": 0.7681552997502585, + "pass@16": 0.8127676135435525, + "pass@32": 0.8456231168387941, + "pass@64": 0.868671287789918, + "pass@128": 0.8859649122807017 + }, + "15": { + "pass@1": 0.403811553030303, + "pass@2": 0.5122766791935102, + "pass@4": 0.6013293119610047, + "pass@8": 0.6697900902768419, + "pass@16": 0.7231940412743582, + "pass@32": 0.7662809711021994, + "pass@64": 0.80072356907216, + "pass@128": 0.8272727272727273 + }, + "16": { + "pass@1": 0.27966969373219375, + "pass@2": 0.3633873690468179, + "pass@4": 0.4406781112830982, + "pass@8": 0.5073770521645653, + "pass@16": 0.5641668062665472, + "pass@32": 0.613915678888368, + "pass@64": 0.6607560737801916, + "pass@128": 0.7122507122507122 + }, + "17": { + "pass@1": 0.25317837732919257, + "pass@2": 0.3383595392967182, + "pass@4": 0.4201410638887531, + "pass@8": 0.49519154029212514, + "pass@16": 0.5608408710318087, + "pass@32": 0.620854047451868, + "pass@64": 0.6752767385641311, + "pass@128": 0.7204968944099379 + }, + "18": { + "pass@1": 0.22141544117647058, + "pass@2": 0.30369311602593757, + "pass@4": 0.38339679442643204, + "pass@8": 0.45511805576024483, + "pass@16": 0.5201182357236519, + "pass@32": 0.5796778115707856, + "pass@64": 0.6310170644117857, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.15997023809523808, + "pass@2": 0.22198690007499064, + "pass@4": 0.28720189496402243, + "pass@8": 0.3504679565583006, + "pass@16": 0.4073951120645691, + "pass@32": 0.45981662672399704, + "pass@64": 0.5113304653878958, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.15187593562874252, + "pass@2": 0.21543293825734353, + "pass@4": 0.2814116064833215, + "pass@8": 0.34650411636039585, + "pass@16": 0.4071150011676281, + "pass@32": 0.4586578382580935, + "pass@64": 0.4999206483403107, + "pass@128": 0.5329341317365269 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 559380, + "answer_accuracy": 0.6982195638280876, + "avg_response_len": 290.8391715929062, + "resp_tokens_sum": 233006384, + "pass_at_k": { + "pass@1": 0.6982195638280876, + "pass@2": 0.7426926682270405, + "pass@4": 0.7791382342535026, + "pass@8": 0.8087130280301855, + "pass@16": 0.8331649486676808, + "pass@32": 0.8543036441538452, + "pass@64": 0.8731143817698518, + "pass@128": 0.8892794376098418 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9144356343283582, + "pass@2": 0.9425960747443888, + "pass@4": 0.9646930267112135, + "pass@8": 0.9811428131208776, + "pass@16": 0.9902786582077016, + "pass@32": 0.9935473106779685, + "pass@64": 0.9940240915063959, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9963839285714285, + "pass@2": 0.9982455708661419, + "pass@4": 0.9992677446569179, + "pass@8": 0.9998301126800612, + "pass@16": 0.9999920274734635, + "pass@32": 0.9999999928369812, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9944651967930029, + "pass@2": 0.9972649703863546, + "pass@4": 0.9991110407846251, + "pass@8": 0.9998379026494393, + "pass@16": 0.9999932937549632, + "pass@32": 0.9999999952610663, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999510751945482, + "pass@4": 0.9999993476692606, + "pass@8": 0.9999999999502821, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9579044117647059, + "pass@2": 0.9702578595414542, + "pass@4": 0.9786070943337964, + "pass@8": 0.9839189372986221, + "pass@16": 0.9879011157481662, + "pass@32": 0.991241607425756, + "pass@64": 0.9932929188162924, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9819844242902208, + "pass@2": 0.9871697943317023, + "pass@4": 0.9902472546807043, + "pass@8": 0.9923964867240731, + "pass@16": 0.9936938955374945, + "pass@32": 0.994448256764124, + "pass@64": 0.9952681007271514, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9788864715189873, + "pass@2": 0.983811624763281, + "pass@4": 0.9868020072451386, + "pass@8": 0.9896018544816012, + "pass@16": 0.9919811742203117, + "pass@32": 0.9937080695528824, + "pass@64": 0.9951619595027914, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9660989114114115, + "pass@2": 0.9772968096048807, + "pass@4": 0.984998759164113, + "pass@8": 0.9914371662868396, + "pass@16": 0.9959159220160351, + "pass@32": 0.9984384250371764, + "pass@64": 0.999630486057751, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9645799839228296, + "pass@2": 0.9815682203711674, + "pass@4": 0.9899260697316372, + "pass@8": 0.9946330963494676, + "pass@16": 0.9979792760818236, + "pass@32": 0.9997288091626138, + "pass@64": 0.9999976591447138, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.913640873015873, + "pass@2": 0.9469831896012993, + "pass@4": 0.9640390475000145, + "pass@8": 0.9737183128947382, + "pass@16": 0.980436931436705, + "pass@32": 0.9851151120888639, + "pass@64": 0.989257268740163, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.84501624617737, + "pass@2": 0.8963995792097086, + "pass@4": 0.9219941976862989, + "pass@8": 0.9370642831438329, + "pass@16": 0.9484829323860189, + "pass@32": 0.9588899261082772, + "pass@64": 0.9691862847869092, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6558641975308642, + "pass@2": 0.7460109695489451, + "pass@4": 0.810327141688307, + "pass@8": 0.853768649246564, + "pass@16": 0.8812443568927503, + "pass@32": 0.8990306053737257, + "pass@64": 0.9133706195283898, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.548219584569733, + "pass@2": 0.6467378764690762, + "pass@4": 0.7249113825534416, + "pass@8": 0.7869967053818288, + "pass@16": 0.8350917715060479, + "pass@32": 0.8689137574921737, + "pass@64": 0.894055173104561, + "pass@128": 0.913946587537092 + }, + "15": { + "pass@1": 0.42896412037037035, + "pass@2": 0.5288854409691839, + "pass@4": 0.609746115358265, + "pass@8": 0.672864986325362, + "pass@16": 0.7236380756477004, + "pass@32": 0.7660735264175047, + "pass@64": 0.7987517046735475, + "pass@128": 0.8240740740740741 + }, + "16": { + "pass@1": 0.3076373525073746, + "pass@2": 0.4040122058857685, + "pass@4": 0.4899784100549376, + "pass@8": 0.5612162942574591, + "pass@16": 0.6209594710613111, + "pass@32": 0.6728396467804595, + "pass@64": 0.7175716835130616, + "pass@128": 0.7551622418879056 + }, + "17": { + "pass@1": 0.24124623493975902, + "pass@2": 0.32365992019258133, + "pass@4": 0.39760372903311775, + "pass@8": 0.4609499419211275, + "pass@16": 0.5210178552073905, + "pass@32": 0.5831115460307031, + "pass@64": 0.6466603576444395, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.2224798387096774, + "pass@2": 0.30349607886715774, + "pass@4": 0.381792796364164, + "pass@8": 0.45000639480557797, + "pass@16": 0.5062970429482582, + "pass@32": 0.5554647423338157, + "pass@64": 0.6031699913032258, + "pass@128": 0.6483870967741936 + }, + "19": { + "pass@1": 0.16901907099697885, + "pass@2": 0.23536334062849376, + "pass@4": 0.30176768170217383, + "pass@8": 0.36282922704048376, + "pass@16": 0.41776046237208814, + "pass@32": 0.4711368236463992, + "pass@64": 0.5235665815669148, + "pass@128": 0.5709969788519638 + }, + "20": { + "pass@1": 0.16356981981981983, + "pass@2": 0.2283974427183088, + "pass@4": 0.2966017620545179, + "pass@8": 0.3632932188833103, + "pass@16": 0.42811660432542914, + "pass@32": 0.49119081476826765, + "pass@64": 0.5479351503093437, + "pass@128": 0.5915915915915916 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6be5fecc3f7b2a5c4a7b92ce264f994b73905262 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38d433aeaad59b6bb02426b1cb159f67aaa41f4bc4758f5025497d90fb8d1679 +size 2616750176 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/summary.csv b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..4a074531ffe486711caa3255b1185a7607c1fb5c --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch1-op9-12,0.9736197916666667,0.4432078125,0.6944555921052632,0.0,0.0,0.0,199.40330902777777,340.78941953125,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/summary.json b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9a2485edbeb5c2d54f3b669687016999ee326c51 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch1-op9-12", + "id_acc": 0.9736197916666667, + "ood_acc": 0.4432078125, + "total_acc": 0.6944555921052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.40330902777777, + "ood_avg_resp_len": 340.78941953125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 560375, + "answer_accuracy": 0.6920533808883971, + "avg_response_len": 260.877005611761, + "resp_tokens_sum": 211239416, + "pass_at_k": { + "pass@1": 0.6920533808883971, + "pass@2": 0.7343789091575517, + "pass@4": 0.7681937301061701, + "pass@8": 0.795565666944629, + "pass@16": 0.8184726146916678, + "pass@32": 0.8382198919038769, + "pass@64": 0.8556873805635508, + "pass@128": 0.8716408472968701 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9248939896755162, + "pass@2": 0.9462058393143337, + "pass@4": 0.9605366295474124, + "pass@8": 0.9713611235954099, + "pass@16": 0.9800066086763779, + "pass@32": 0.9868547832437569, + "pass@64": 0.9923681900521832, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9971524532710281, + "pass@2": 0.997739056712537, + "pass@4": 0.9983754203621744, + "pass@8": 0.9991721938425355, + "pass@16": 0.9998009003384084, + "pass@32": 0.9999920326399729, + "pass@64": 0.9999999987627288, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975391819571865, + "pass@2": 0.9994220905873004, + "pass@4": 0.9999528238442673, + "pass@8": 0.9999996971931873, + "pass@16": 0.9999999999944118, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9949462890625, + "pass@2": 0.9967554287647638, + "pass@4": 0.9983901592379079, + "pass@8": 0.9995789226623526, + "pass@16": 0.9999708149888308, + "pass@32": 0.9999999057416857, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9621031746031746, + "pass@2": 0.9698705630546183, + "pass@4": 0.9753851185268506, + "pass@8": 0.9798042600934502, + "pass@16": 0.9836601066054623, + "pass@32": 0.9872888783520354, + "pass@64": 0.9907961636655847, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9682354041916168, + "pass@2": 0.9816244725116692, + "pass@4": 0.9886694962156675, + "pass@8": 0.992154678669665, + "pass@16": 0.9939489144577033, + "pass@32": 0.994741302662305, + "pass@64": 0.995508975120257, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9770247781065089, + "pass@2": 0.9843062042817871, + "pass@4": 0.9889038042766843, + "pass@8": 0.9927051296156951, + "pass@16": 0.9955222820647343, + "pass@32": 0.9972433727539783, + "pass@64": 0.9984797932474441, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9729817708333334, + "pass@2": 0.9827751962194632, + "pass@4": 0.9881974047114304, + "pass@8": 0.9926713618110146, + "pass@16": 0.9959623220314486, + "pass@32": 0.9974855169110697, + "pass@64": 0.9983969843073067, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9644366197183099, + "pass@2": 0.9820408672507485, + "pass@4": 0.9905973869463501, + "pass@8": 0.9953054121741431, + "pass@16": 0.9980914371290873, + "pass@32": 0.9993323812044371, + "pass@64": 0.9999188136773991, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8889429886685553, + "pass@2": 0.9242913664651687, + "pass@4": 0.9465331700747036, + "pass@8": 0.9604262141604022, + "pass@16": 0.9689611062295932, + "pass@32": 0.9749640416956897, + "pass@64": 0.9803316295352276, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8323181497175142, + "pass@2": 0.8870070399039109, + "pass@4": 0.9188809555585208, + "pass@8": 0.9380959870647928, + "pass@16": 0.9504149057699475, + "pass@32": 0.9579377264267592, + "pass@64": 0.9628166202895165, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.6575480015673981, + "pass@2": 0.7480878051489646, + "pass@4": 0.8102339428378657, + "pass@8": 0.8522237777096143, + "pass@16": 0.882257665109769, + "pass@32": 0.9087626399289626, + "pass@64": 0.93377578450576, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.5357768691588785, + "pass@2": 0.6224040737606394, + "pass@4": 0.6857702331601073, + "pass@8": 0.7324715141602004, + "pass@16": 0.7720395421287615, + "pass@32": 0.8098585396648957, + "pass@64": 0.8454894274337638, + "pass@128": 0.8691588785046729 + }, + "15": { + "pass@1": 0.4288972182080925, + "pass@2": 0.5222473658458879, + "pass@4": 0.5974337641501454, + "pass@8": 0.6592894122612182, + "pass@16": 0.7139905030261611, + "pass@32": 0.76220652456853, + "pass@64": 0.8015069468721714, + "pass@128": 0.8323699421965318 + }, + "16": { + "pass@1": 0.27419354838709675, + "pass@2": 0.3570941548133095, + "pass@4": 0.4288660543440137, + "pass@8": 0.48969756116313695, + "pass@16": 0.5435783504560074, + "pass@32": 0.5964149406645595, + "pass@64": 0.6502359398435504, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.24882586705202311, + "pass@2": 0.32303341620772813, + "pass@4": 0.3890596267590826, + "pass@8": 0.4457163994015934, + "pass@16": 0.4957672003720015, + "pass@32": 0.5433302498892872, + "pass@64": 0.5907082354561285, + "pass@128": 0.638728323699422 + }, + "18": { + "pass@1": 0.21283482142857144, + "pass@2": 0.28537085208098983, + "pass@4": 0.3563121484814401, + "pass@8": 0.42147138400766687, + "pass@16": 0.4769469684933736, + "pass@32": 0.5212049806482858, + "pass@64": 0.5583199932477526, + "pass@128": 0.5942857142857143 + }, + "19": { + "pass@1": 0.16884853603603603, + "pass@2": 0.23270058936889648, + "pass@4": 0.29553178206327807, + "pass@8": 0.35391545800133395, + "pass@16": 0.4071753895217641, + "pass@32": 0.45166801653626926, + "pass@64": 0.48833945361368375, + "pass@128": 0.5285285285285285 + }, + "20": { + "pass@1": 0.17027965465465467, + "pass@2": 0.23516528339362977, + "pass@4": 0.3000875284733552, + "pass@8": 0.3593155322698133, + "pass@16": 0.4107097007726869, + "pass@32": 0.45401753885710516, + "pass@64": 0.4883729600405135, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 569161, + "answer_accuracy": 0.69315203624318, + "avg_response_len": 269.96939424201094, + "resp_tokens_sum": 221677269, + "pass_at_k": { + "pass@1": 0.69315203624318, + "pass@2": 0.7366847701008297, + "pass@4": 0.7730375837470743, + "pass@8": 0.8032359196001282, + "pass@16": 0.828144989935398, + "pass@32": 0.84898641615469, + "pass@64": 0.8667552542800026, + "pass@128": 0.8826188620420888 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9205329754601227, + "pass@2": 0.9431817152553016, + "pass@4": 0.9587725736736894, + "pass@8": 0.9698210443057249, + "pass@16": 0.9783974902701662, + "pass@32": 0.9842556829652028, + "pass@64": 0.9868527944754925, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9962481003039514, + "pass@2": 0.9974010063901586, + "pass@4": 0.9979578029570011, + "pass@8": 0.9986493669171034, + "pass@16": 0.9994341559274335, + "pass@32": 0.9999199744887792, + "pass@64": 0.9999995792634084, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9943418560606061, + "pass@2": 0.996974916487712, + "pass@4": 0.9986633554328435, + "pass@8": 0.9996870700423274, + "pass@16": 0.9999813836429393, + "pass@32": 0.999999959369951, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9977579511834319, + "pass@2": 0.9993528106508875, + "pass@4": 0.9998821282694692, + "pass@8": 0.9999960334397089, + "pass@16": 0.9999999974655996, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9718523550724638, + "pass@2": 0.9773992924797448, + "pass@4": 0.9825246996299374, + "pass@8": 0.9874108605762586, + "pass@16": 0.9912208543323465, + "pass@32": 0.9941367591421882, + "pass@64": 0.9967430652968408, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9824050859598854, + "pass@2": 0.9914463145545203, + "pass@4": 0.9956285275407906, + "pass@8": 0.9968915638760587, + "pass@16": 0.9971227705571397, + "pass@32": 0.9971346536923668, + "pass@64": 0.9971346704871058, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9837653540462428, + "pass@2": 0.9907264120886624, + "pass@4": 0.9944128431742274, + "pass@8": 0.9963867706256992, + "pass@16": 0.9970495251707892, + "pass@32": 0.9971095689707938, + "pass@64": 0.9971098265895346, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9706866197183098, + "pass@2": 0.987252550737496, + "pass@4": 0.9947429746457749, + "pass@8": 0.9977459292370181, + "pass@16": 0.999301853323875, + "pass@32": 0.9999206896276759, + "pass@64": 0.9999996096160376, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9561190119760479, + "pass@2": 0.9741652276156351, + "pass@4": 0.9849497023949854, + "pass@8": 0.9913124752168228, + "pass@16": 0.9944909189241692, + "pass@32": 0.9962477073890863, + "pass@64": 0.9969666351508508, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9085325677710844, + "pass@2": 0.9393202732188596, + "pass@4": 0.9578793574184248, + "pass@8": 0.9694906501998141, + "pass@16": 0.9766675676967038, + "pass@32": 0.9821868207145709, + "pass@64": 0.9872097467394703, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8305005877742947, + "pass@2": 0.8724626200232023, + "pass@4": 0.8967490576217158, + "pass@8": 0.914954573017507, + "pass@16": 0.9291971281957504, + "pass@32": 0.9389488700907844, + "pass@64": 0.9468632868540318, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6433385854341737, + "pass@2": 0.7310224778887932, + "pass@4": 0.7918085777618122, + "pass@8": 0.8355008886669074, + "pass@16": 0.8664531704460665, + "pass@32": 0.8875200988088843, + "pass@64": 0.9043131635172054, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5319581505847953, + "pass@2": 0.6288834064327484, + "pass@4": 0.7076227626151991, + "pass@8": 0.7681552997502585, + "pass@16": 0.8127676135435525, + "pass@32": 0.8456231168387941, + "pass@64": 0.868671287789918, + "pass@128": 0.8859649122807017 + }, + "15": { + "pass@1": 0.403811553030303, + "pass@2": 0.5122766791935102, + "pass@4": 0.6013293119610047, + "pass@8": 0.6697900902768419, + "pass@16": 0.7231940412743582, + "pass@32": 0.7662809711021994, + "pass@64": 0.80072356907216, + "pass@128": 0.8272727272727273 + }, + "16": { + "pass@1": 0.27966969373219375, + "pass@2": 0.3633873690468179, + "pass@4": 0.4406781112830982, + "pass@8": 0.5073770521645653, + "pass@16": 0.5641668062665472, + "pass@32": 0.613915678888368, + "pass@64": 0.6607560737801916, + "pass@128": 0.7122507122507122 + }, + "17": { + "pass@1": 0.25317837732919257, + "pass@2": 0.3383595392967182, + "pass@4": 0.4201410638887531, + "pass@8": 0.49519154029212514, + "pass@16": 0.5608408710318087, + "pass@32": 0.620854047451868, + "pass@64": 0.6752767385641311, + "pass@128": 0.7204968944099379 + }, + "18": { + "pass@1": 0.22141544117647058, + "pass@2": 0.30369311602593757, + "pass@4": 0.38339679442643204, + "pass@8": 0.45511805576024483, + "pass@16": 0.5201182357236519, + "pass@32": 0.5796778115707856, + "pass@64": 0.6310170644117857, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.15997023809523808, + "pass@2": 0.22198690007499064, + "pass@4": 0.28720189496402243, + "pass@8": 0.3504679565583006, + "pass@16": 0.4073951120645691, + "pass@32": 0.45981662672399704, + "pass@64": 0.5113304653878958, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.15187593562874252, + "pass@2": 0.21543293825734353, + "pass@4": 0.2814116064833215, + "pass@8": 0.34650411636039585, + "pass@16": 0.4071150011676281, + "pass@32": 0.4586578382580935, + "pass@64": 0.4999206483403107, + "pass@128": 0.5329341317365269 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 559380, + "answer_accuracy": 0.6982195638280876, + "avg_response_len": 290.8391715929062, + "resp_tokens_sum": 233006384, + "pass_at_k": { + "pass@1": 0.6982195638280876, + "pass@2": 0.7426926682270405, + "pass@4": 0.7791382342535026, + "pass@8": 0.8087130280301855, + "pass@16": 0.8331649486676808, + "pass@32": 0.8543036441538452, + "pass@64": 0.8731143817698518, + "pass@128": 0.8892794376098418 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9144356343283582, + "pass@2": 0.9425960747443888, + "pass@4": 0.9646930267112135, + "pass@8": 0.9811428131208776, + "pass@16": 0.9902786582077016, + "pass@32": 0.9935473106779685, + "pass@64": 0.9940240915063959, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9963839285714285, + "pass@2": 0.9982455708661419, + "pass@4": 0.9992677446569179, + "pass@8": 0.9998301126800612, + "pass@16": 0.9999920274734635, + "pass@32": 0.9999999928369812, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9944651967930029, + "pass@2": 0.9972649703863546, + "pass@4": 0.9991110407846251, + "pass@8": 0.9998379026494393, + "pass@16": 0.9999932937549632, + "pass@32": 0.9999999952610663, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999510751945482, + "pass@4": 0.9999993476692606, + "pass@8": 0.9999999999502821, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9579044117647059, + "pass@2": 0.9702578595414542, + "pass@4": 0.9786070943337964, + "pass@8": 0.9839189372986221, + "pass@16": 0.9879011157481662, + "pass@32": 0.991241607425756, + "pass@64": 0.9932929188162924, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9819844242902208, + "pass@2": 0.9871697943317023, + "pass@4": 0.9902472546807043, + "pass@8": 0.9923964867240731, + "pass@16": 0.9936938955374945, + "pass@32": 0.994448256764124, + "pass@64": 0.9952681007271514, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9788864715189873, + "pass@2": 0.983811624763281, + "pass@4": 0.9868020072451386, + "pass@8": 0.9896018544816012, + "pass@16": 0.9919811742203117, + "pass@32": 0.9937080695528824, + "pass@64": 0.9951619595027914, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9660989114114115, + "pass@2": 0.9772968096048807, + "pass@4": 0.984998759164113, + "pass@8": 0.9914371662868396, + "pass@16": 0.9959159220160351, + "pass@32": 0.9984384250371764, + "pass@64": 0.999630486057751, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9645799839228296, + "pass@2": 0.9815682203711674, + "pass@4": 0.9899260697316372, + "pass@8": 0.9946330963494676, + "pass@16": 0.9979792760818236, + "pass@32": 0.9997288091626138, + "pass@64": 0.9999976591447138, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.913640873015873, + "pass@2": 0.9469831896012993, + "pass@4": 0.9640390475000145, + "pass@8": 0.9737183128947382, + "pass@16": 0.980436931436705, + "pass@32": 0.9851151120888639, + "pass@64": 0.989257268740163, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.84501624617737, + "pass@2": 0.8963995792097086, + "pass@4": 0.9219941976862989, + "pass@8": 0.9370642831438329, + "pass@16": 0.9484829323860189, + "pass@32": 0.9588899261082772, + "pass@64": 0.9691862847869092, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6558641975308642, + "pass@2": 0.7460109695489451, + "pass@4": 0.810327141688307, + "pass@8": 0.853768649246564, + "pass@16": 0.8812443568927503, + "pass@32": 0.8990306053737257, + "pass@64": 0.9133706195283898, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.548219584569733, + "pass@2": 0.6467378764690762, + "pass@4": 0.7249113825534416, + "pass@8": 0.7869967053818288, + "pass@16": 0.8350917715060479, + "pass@32": 0.8689137574921737, + "pass@64": 0.894055173104561, + "pass@128": 0.913946587537092 + }, + "15": { + "pass@1": 0.42896412037037035, + "pass@2": 0.5288854409691839, + "pass@4": 0.609746115358265, + "pass@8": 0.672864986325362, + "pass@16": 0.7236380756477004, + "pass@32": 0.7660735264175047, + "pass@64": 0.7987517046735475, + "pass@128": 0.8240740740740741 + }, + "16": { + "pass@1": 0.3076373525073746, + "pass@2": 0.4040122058857685, + "pass@4": 0.4899784100549376, + "pass@8": 0.5612162942574591, + "pass@16": 0.6209594710613111, + "pass@32": 0.6728396467804595, + "pass@64": 0.7175716835130616, + "pass@128": 0.7551622418879056 + }, + "17": { + "pass@1": 0.24124623493975902, + "pass@2": 0.32365992019258133, + "pass@4": 0.39760372903311775, + "pass@8": 0.4609499419211275, + "pass@16": 0.5210178552073905, + "pass@32": 0.5831115460307031, + "pass@64": 0.6466603576444395, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.2224798387096774, + "pass@2": 0.30349607886715774, + "pass@4": 0.381792796364164, + "pass@8": 0.45000639480557797, + "pass@16": 0.5062970429482582, + "pass@32": 0.5554647423338157, + "pass@64": 0.6031699913032258, + "pass@128": 0.6483870967741936 + }, + "19": { + "pass@1": 0.16901907099697885, + "pass@2": 0.23536334062849376, + "pass@4": 0.30176768170217383, + "pass@8": 0.36282922704048376, + "pass@16": 0.41776046237208814, + "pass@32": 0.4711368236463992, + "pass@64": 0.5235665815669148, + "pass@128": 0.5709969788519638 + }, + "20": { + "pass@1": 0.16356981981981983, + "pass@2": 0.2283974427183088, + "pass@4": 0.2966017620545179, + "pass@8": 0.3632932188833103, + "pass@16": 0.42811660432542914, + "pass@32": 0.49119081476826765, + "pass@64": 0.5479351503093437, + "pass@128": 0.5915915915915916 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..63524e0261025fb2ea1439d5c180a3298f35328e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2efec8533bf6eaeae381a78eeff9a7f4a57b228d63dd7221eda299b62b9ea0b9 +size 1789389163 diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fd28deaabdf91341574382966758f7fda9890be2 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op7-10", + "id": { + "answer_accuracy": 0.9930243055555555, + "avg_response_len": 200.27195746527778, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9930243055555555, + "pass@2": 0.9949065644138231, + "pass@4": 0.996174514216973, + "pass@8": 0.9970333205879605, + "pass@16": 0.9976473464798407, + "pass@32": 0.998138523214659, + "pass@64": 0.9985803253764225, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954296875, + "pass@2": 0.9661904527559058, + "pass@4": 0.9746338377390329, + "pass@8": 0.9803727820809247, + "pass@16": 0.984347680858482, + "pass@32": 0.9872299909032018, + "pass@64": 0.9893549245650947, + "pass@128": 0.991 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997109375, + "pass@2": 0.999918061023622, + "pass@4": 0.9999938090551181, + "pass@8": 0.9999999729957695, + "pass@16": 0.999999999999862, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.995140625, + "pass@2": 0.9963550688976379, + "pass@4": 0.997230504030746, + "pass@8": 0.9979210739667388, + "pass@16": 0.998400288705097, + "pass@32": 0.998829650243074, + "pass@64": 0.9993779525919387, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991328125, + "pass@2": 0.9993145915354331, + "pass@4": 0.9995343371766029, + "pass@8": 0.9997890613208846, + "pass@16": 0.9999604948521619, + "pass@32": 0.9999991088495628, + "pass@64": 0.9999999999731547, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977578125, + "pass@2": 0.9984231053149607, + "pass@4": 0.9988245402137234, + "pass@8": 0.9990286195412216, + "pass@16": 0.9991241570481809, + "pass@32": 0.9992499998305313, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.996796875, + "pass@2": 0.9983270177165355, + "pass@4": 0.9989689832208474, + "pass@8": 0.9991198680433677, + "pass@16": 0.9992352358817035, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943828125, + "pass@2": 0.995630782480315, + "pass@4": 0.9963846165166853, + "pass@8": 0.9970685073427377, + "pass@16": 0.9977582609730773, + "pass@32": 0.9984989827276101, + "pass@64": 0.999238082753677, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.954296875, + "2": 1.0, + "3": 1.0, + "4": 0.9997109375, + "5": 0.995140625, + "6": 0.9991328125, + "7": 0.9977578125, + "8": 0.996796875, + "9": 0.9943828125 + }, + "per_op_avg_response_len": { + "10": 281.391109375, + "2": 115.720453125, + "3": 144.8661328125, + "4": 160.0339921875, + "5": 173.204265625, + "6": 199.208125, + "7": 224.3385, + "8": 243.8726953125, + "9": 259.81234375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.391109375, + "2": 115.720453125, + "3": 144.8661328125, + "4": 160.0339921875, + "5": 173.204265625, + "6": 199.208125, + "7": 224.3385, + "8": 243.8726953125, + "9": 259.81234375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376735, + "answer_accuracy": 0.9940027651131375, + "avg_response_len": 190.79767973235394, + "resp_tokens_sum": 72313847, + "pass_at_k": { + "pass@1": 0.9940027651131375, + "pass@2": 0.9958728147678351, + "pass@4": 0.9970037103708035, + "pass@8": 0.997725089213169, + "pass@16": 0.9982333962357546, + "pass@32": 0.9986601785310285, + "pass@64": 0.9990424727659456, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.963472529498525, + "pass@2": 0.9744087229693631, + "pass@4": 0.9803487268737424, + "pass@8": 0.9836354699131256, + "pass@16": 0.9861514017924913, + "pass@32": 0.9888094834427503, + "pass@64": 0.9916772440992451, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990966796875, + "pass@2": 0.9997439406988189, + "pass@4": 0.9999806532972441, + "pass@8": 0.9999999156117797, + "pass@16": 0.9999999999995689, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951388888888889, + "pass@2": 0.9965351987251593, + "pass@4": 0.9979581129739735, + "pass@8": 0.9991604105838077, + "pass@16": 0.999826156800224, + "pass@32": 0.9999942644471409, + "pass@64": 0.9999999994794706, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974270209580839, + "pass@2": 0.997947878848602, + "pass@4": 0.9986057999299488, + "pass@8": 0.9993684470685168, + "pass@16": 0.9998817211142573, + "pass@32": 0.9999973318849184, + "pass@64": 0.9999999999196246, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976423816568047, + "pass@2": 0.9986568455015609, + "pass@4": 0.9994379571821275, + "pass@8": 0.9999004126844195, + "pass@16": 0.999997506107353, + "pass@32": 0.9999999994986136, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9992487980769231, + "pass@2": 0.9998986567484354, + "pass@4": 0.9999973395753415, + "pass@8": 0.9999999989008069, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953125, + "pass@2": 0.996617500277254, + "pass@4": 0.9974524414201746, + "pass@8": 0.9980863720132519, + "pass@16": 0.9987573795437611, + "pass@32": 0.9995185120065893, + "pass@64": 0.999961060003192, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387424, + "answer_accuracy": 0.9917267365661862, + "avg_response_len": 198.29351654652686, + "resp_tokens_sum": 77464552, + "pass_at_k": { + "pass@1": 0.9917267365661862, + "pass@2": 0.9935355656159379, + "pass@4": 0.9948847051064882, + "pass@8": 0.9958047355370226, + "pass@16": 0.9964492963271934, + "pass@32": 0.9970505110300054, + "pass@64": 0.997730272723979, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9395609662576687, + "pass@2": 0.9548636690256509, + "pass@4": 0.966663017487078, + "pass@8": 0.9743895360249896, + "pass@16": 0.9788663525185067, + "pass@32": 0.981713951963166, + "pass@64": 0.9837144251220994, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9971693840579711, + "pass@2": 0.9972362490014836, + "pass@4": 0.9973668033343658, + "pass@8": 0.997615447525581, + "pass@16": 0.9980647799459851, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999776146131805, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9971324060693642, + "pass@2": 0.997154985549133, + "pass@4": 0.9972001445086706, + "pass@8": 0.9972904624277457, + "pass@16": 0.997471098265896, + "pass@32": 0.9978323699421965, + "pass@64": 0.9985549132947977, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959947183098592, + "pass@2": 0.9968451951868693, + "pass@4": 0.997296264463421, + "pass@8": 0.9975244495786237, + "pass@16": 0.9978457356837299, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9941401645527842, + "pass@4": 0.994290434878275, + "pass@8": 0.994563039275777, + "pass@16": 0.9950905276062049, + "pass@32": 0.9960747795747089, + "pass@64": 0.9977603847423264, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379805, + "answer_accuracy": 0.9933801682289923, + "avg_response_len": 211.68526113157014, + "resp_tokens_sum": 80934896, + "pass_at_k": { + "pass@1": 0.9933801682289923, + "pass@2": 0.9953495577291624, + "pass@4": 0.9966704123065969, + "pass@8": 0.9976028936298824, + "pass@16": 0.9982905188061268, + "pass@32": 0.9987330969661806, + "pass@64": 0.9989907513137772, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9593516791044776, + "pass@2": 0.9688965360206839, + "pass@4": 0.9766073899344672, + "pass@8": 0.9828936389141163, + "pass@16": 0.9878564917307295, + "pass@32": 0.9909994856003504, + "pass@64": 0.9924938394795402, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9930836397058823, + "pass@2": 0.9952940452755904, + "pass@4": 0.9964180920399656, + "pass@8": 0.9970829859897469, + "pass@16": 0.9974197065637099, + "pass@32": 0.9977941128663109, + "pass@64": 0.9985294117647059, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985660601265823, + "pass@2": 0.9995616029602312, + "pass@4": 0.9999470274878615, + "pass@8": 0.9999993039679993, + "pass@16": 0.9999999999490367, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953547297297297, + "pass@2": 0.9984342117708261, + "pass@4": 0.9997887068733525, + "pass@8": 0.9999960624201342, + "pass@16": 0.9999999991410792, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941971463022508, + "pass@2": 0.9961053212902246, + "pass@4": 0.9974147735150468, + "pass@8": 0.9985973960126167, + "pass@16": 0.9994828135516711, + "pass@32": 0.9999386964222448, + "pass@64": 0.9999998004778354, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4938328125, + "avg_response_len": 337.38122890625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4938328125, + "pass@2": 0.5599698695866046, + "pass@4": 0.6156492943288245, + "pass@8": 0.6620199241379036, + "pass@16": 0.6999616205128099, + "pass@32": 0.7315920462120327, + "pass@64": 0.7590439757988445, + "pass@128": 0.7827 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9493203125, + "pass@2": 0.9654307332677168, + "pass@4": 0.9747091845706785, + "pass@8": 0.9807236154980903, + "pass@16": 0.9846030327276756, + "pass@32": 0.9876037567845825, + "pass@64": 0.9906123675412761, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.8656640625, + "pass@2": 0.8981660925196859, + "pass@4": 0.9162192402512177, + "pass@8": 0.9294925566724376, + "pass@16": 0.9403418552190832, + "pass@32": 0.9504850673157726, + "pass@64": 0.9600327044081675, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.7539765625, + "pass@2": 0.8214111712598431, + "pass@4": 0.8642939338207711, + "pass@8": 0.8939748081673056, + "pass@16": 0.9152320369752246, + "pass@32": 0.931471224927995, + "pass@64": 0.9453016174913549, + "pass@128": 0.957 + }, + "14": { + "pass@1": 0.66509375, + "pass@2": 0.7435506889763775, + "pass@4": 0.7978263482377215, + "pass@8": 0.8370823461640506, + "pass@16": 0.8670369761030744, + "pass@32": 0.8905706816269232, + "pass@64": 0.909815526205601, + "pass@128": 0.924 + }, + "15": { + "pass@1": 0.5480078125, + "pass@2": 0.6441775344488198, + "pass@4": 0.7138529732845889, + "pass@8": 0.7638165706839227, + "pass@16": 0.8011898814003716, + "pass@32": 0.8304545822003983, + "pass@64": 0.8536875341305782, + "pass@128": 0.873 + }, + "16": { + "pass@1": 0.3315, + "pass@2": 0.4212034940944888, + "pass@4": 0.5018796497937756, + "pass@8": 0.569041091355488, + "pass@16": 0.6232936592313723, + "pass@32": 0.6689611183103249, + "pass@64": 0.7101747586666186, + "pass@128": 0.748 + }, + "17": { + "pass@1": 0.264140625, + "pass@2": 0.34535888287401634, + "pass@4": 0.42405996878515173, + "pass@8": 0.493988943111484, + "pass@16": 0.5533799983646059, + "pass@32": 0.6040125546424108, + "pass@64": 0.6488027969675257, + "pass@128": 0.688 + }, + "18": { + "pass@1": 0.2277890625, + "pass@2": 0.3007801427165356, + "pass@4": 0.37204404443194605, + "pass@8": 0.435300861022257, + "pass@16": 0.48862231997077116, + "pass@32": 0.5342048573239302, + "pass@64": 0.5734251003614904, + "pass@128": 0.609 + }, + "19": { + "pass@1": 0.1720546875, + "pass@2": 0.2371135580708661, + "pass@4": 0.30388390813648286, + "pass@8": 0.3649321499143648, + "pass@16": 0.4171717877808307, + "pass@32": 0.4631558514273401, + "pass@64": 0.506340857897091, + "pass@128": 0.546 + }, + "20": { + "pass@1": 0.16078125, + "pass@2": 0.2225063976377949, + "pass@4": 0.2877236919760029, + "pass@8": 0.35184629878963847, + "pass@16": 0.40874465735495447, + "pass@32": 0.45500076756063307, + "pass@64": 0.49224649431868345, + "pass@128": 0.522 + } + }, + "per_op_accuracy": { + "11": 0.9493203125, + "12": 0.8656640625, + "13": 0.7539765625, + "14": 0.66509375, + "15": 0.5480078125, + "16": 0.3315, + "17": 0.264140625, + "18": 0.2277890625, + "19": 0.1720546875, + "20": 0.16078125 + }, + "per_op_avg_response_len": { + "11": 298.9996796875, + "12": 308.2072109375, + "13": 324.847875, + "14": 344.7609296875, + "15": 353.43428125, + "16": 337.8661484375, + "17": 352.958828125, + "18": 356.199546875, + "19": 343.0367578125, + "20": 353.50103125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.9996796875, + "12": 308.2072109375, + "13": 324.847875, + "14": 344.7609296875, + "15": 353.43428125, + "16": 337.8661484375, + "17": 352.958828125, + "18": 356.199546875, + "19": 343.0367578125, + "20": 353.50103125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 212384, + "answer_accuracy": 0.49309063893016347, + "avg_response_len": 320.34922223254085, + "resp_tokens_sum": 137980817, + "pass_at_k": { + "pass@1": 0.49309063893016347, + "pass@2": 0.5591938347509696, + "pass@4": 0.6148491952157553, + "pass@8": 0.6611058483132114, + "pass@16": 0.6988262477133611, + "pass@32": 0.7299767141899479, + "pass@64": 0.7570822013847831, + "pass@128": 0.7806835066864785 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9400230169971672, + "pass@2": 0.9581030704200219, + "pass@4": 0.9695777809997548, + "pass@8": 0.9776203275632077, + "pass@16": 0.9829647487484047, + "pass@32": 0.9867373748424951, + "pass@64": 0.990728249665513, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.859816384180791, + "pass@2": 0.8918441934694605, + "pass@4": 0.9098151451937151, + "pass@8": 0.9241309150223803, + "pass@16": 0.9363586934573225, + "pass@32": 0.9478360071414645, + "pass@64": 0.9589968897614609, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7661637931034483, + "pass@2": 0.8327435020857502, + "pass@4": 0.8767341852644595, + "pass@8": 0.9089967422970507, + "pass@16": 0.9321636362076665, + "pass@32": 0.94779652546968, + "pass@64": 0.9595366262742799, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6715342679127726, + "pass@2": 0.7542328967056693, + "pass@4": 0.8122993729171699, + "pass@8": 0.8509755820397881, + "pass@16": 0.8762950400746685, + "pass@32": 0.8949873380135102, + "pass@64": 0.9119272070496415, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5707640895953757, + "pass@2": 0.6658713474580124, + "pass@4": 0.7362232691469921, + "pass@8": 0.7895324052420507, + "pass@16": 0.829999658967477, + "pass@32": 0.8602938952993778, + "pass@64": 0.8807124344185117, + "pass@128": 0.8930635838150289 + }, + "16": { + "pass@1": 0.3118951612903226, + "pass@2": 0.40475576263652524, + "pass@4": 0.4862871569481233, + "pass@8": 0.554336074823824, + "pass@16": 0.6110271182506712, + "pass@32": 0.6587839935388526, + "pass@64": 0.7001171727816106, + "pass@128": 0.7387096774193549 + }, + "17": { + "pass@1": 0.244829299132948, + "pass@2": 0.328066328910837, + "pass@4": 0.41020303468208086, + "pass@8": 0.48089300913906174, + "pass@16": 0.5372670405574277, + "pass@32": 0.5825153543250646, + "pass@64": 0.621207883664678, + "pass@128": 0.653179190751445 + }, + "18": { + "pass@1": 0.20457589285714287, + "pass@2": 0.2681696428571427, + "pass@4": 0.3305764103594193, + "pass@8": 0.3901694784094153, + "pass@16": 0.44529528231015914, + "pass@32": 0.494447129452626, + "pass@64": 0.5396755944182013, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.17171077327327328, + "pass@2": 0.24016523905795556, + "pass@4": 0.30821597384786364, + "pass@8": 0.3693761252170798, + "pass@16": 0.42273905856526567, + "pass@32": 0.47019284079490814, + "pass@64": 0.5157973767607983, + "pass@128": 0.5615615615615616 + }, + "20": { + "pass@1": 0.1664085960960961, + "pass@2": 0.2306275271334326, + "pass@4": 0.2956622741977073, + "pass@8": 0.355173554615156, + "pass@16": 0.4063776034385708, + "pass@32": 0.4498967766195675, + "pass@64": 0.4870903338771562, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 212251, + "answer_accuracy": 0.4930749145108534, + "avg_response_len": 332.6185534678858, + "resp_tokens_sum": 143180313, + "pass_at_k": { + "pass@1": 0.4930749145108534, + "pass@2": 0.5592165992353123, + "pass@4": 0.6151298058081739, + "pass@8": 0.6618345267918192, + "pass@16": 0.6995971721119053, + "pass@32": 0.7308501365465514, + "pass@64": 0.7582547656694086, + "pass@128": 0.7820398453761522 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9549134036144579, + "pass@2": 0.9673173501090979, + "pass@4": 0.9744871380911725, + "pass@8": 0.979511684978502, + "pass@16": 0.9829385174486236, + "pass@32": 0.9856195234202649, + "pass@64": 0.9875988046336269, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8561422413793104, + "pass@2": 0.8865071088292648, + "pass@4": 0.9036359111067229, + "pass@8": 0.9166247203980661, + "pass@16": 0.9272387717823719, + "pass@32": 0.9379651890883318, + "pass@64": 0.9487072364211467, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.7606573879551821, + "pass@2": 0.8298419269282518, + "pass@4": 0.8707777709508995, + "pass@8": 0.896675537557301, + "pass@16": 0.9137837552594845, + "pass@32": 0.9273137994362725, + "pass@64": 0.9404390148244085, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.6662097953216374, + "pass@2": 0.7445089100704514, + "pass@4": 0.7975902315938572, + "pass@8": 0.8359411040303549, + "pass@16": 0.8656772291606879, + "pass@32": 0.8890270125392543, + "pass@64": 0.9079604648318258, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5242897727272727, + "pass@2": 0.6281238815318541, + "pass@4": 0.7041429557100817, + "pass@8": 0.7557992599136547, + "pass@16": 0.792250838503205, + "pass@32": 0.8210298940378924, + "pass@64": 0.848988614620591, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.33751780626780625, + "pass@2": 0.42342638860847526, + "pass@4": 0.5021861944073225, + "pass@8": 0.5700196165941456, + "pass@16": 0.6257757067875516, + "pass@32": 0.6727157231894736, + "pass@64": 0.7144038374122713, + "pass@128": 0.7492877492877493 + }, + "17": { + "pass@1": 0.27139945652173914, + "pass@2": 0.3544044511419766, + "pass@4": 0.43338719558347116, + "pass@8": 0.5018487096383886, + "pass@16": 0.5594525238622866, + "pass@32": 0.6109768176878541, + "pass@64": 0.6613991318224222, + "pass@128": 0.7080745341614907 + }, + "18": { + "pass@1": 0.24184283088235295, + "pass@2": 0.32053576597962, + "pass@4": 0.39809695130020534, + "pass@8": 0.46780903740778607, + "pass@16": 0.5253409135401563, + "pass@32": 0.5743579603286321, + "pass@64": 0.6150981652884812, + "pass@128": 0.6470588235294118 + }, + "19": { + "pass@1": 0.17094494047619047, + "pass@2": 0.23047607330333703, + "pass@4": 0.2933763884983127, + "pass@8": 0.35234886890680117, + "pass@16": 0.40309559472804163, + "pass@32": 0.44648802339630994, + "pass@64": 0.4845569241923335, + "pass@128": 0.5178571428571429 + }, + "20": { + "pass@1": 0.15009824101796407, + "pass@2": 0.2077268936536376, + "pass@4": 0.27286165099497306, + "pass@8": 0.3401589985741033, + "pass@16": 0.39834828717305787, + "pass@32": 0.440763071508351, + "pass@64": 0.4712399556727392, + "pass@128": 0.4940119760479042 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 207471, + "answer_accuracy": 0.49537505730440096, + "avg_response_len": 359.7924697241748, + "resp_tokens_sum": 150686843, + "pass_at_k": { + "pass@1": 0.49537505730440096, + "pass@2": 0.5615421817545219, + "pass@4": 0.6170060710435342, + "pass@8": 0.6631505342922434, + "pass@16": 0.7015038483374672, + "pass@32": 0.7340158342497117, + "pass@64": 0.7618726691266309, + "pass@128": 0.7854523227383863 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.953844246031746, + "pass@2": 0.9716539338832647, + "pass@4": 0.980693644544432, + "pass@8": 0.9854786046203673, + "pass@16": 0.9881932972271286, + "pass@32": 0.990665973617949, + "pass@64": 0.9936587056221776, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.8812834480122325, + "pass@2": 0.9163837318500325, + "pass@4": 0.9354275939131463, + "pass@8": 0.9478499295031567, + "pass@16": 0.9574364206012684, + "pass@32": 0.9655664387416397, + "pass@64": 0.9722024373524911, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7346161265432098, + "pass@2": 0.8009642783367357, + "pass@4": 0.8449014336170942, + "pass@8": 0.8762088903289816, + "pass@16": 0.9001575197448859, + "pass@32": 0.919978725013635, + "pass@64": 0.9366441525232889, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6578264094955489, + "pass@2": 0.7324032103553814, + "pass@4": 0.7842800899887509, + "pass@8": 0.8250069043053341, + "pass@16": 0.859598385359498, + "pass@32": 0.8879302903150209, + "pass@64": 0.9096866877453451, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.5478636188271605, + "pass@2": 0.6373616275396131, + "pass@4": 0.6998535394302561, + "pass@8": 0.7445203786995883, + "pass@16": 0.7795284650974312, + "pass@32": 0.8081883623281141, + "pass@64": 0.8296135462869662, + "pass@128": 0.845679012345679 + }, + "16": { + "pass@1": 0.34319690265486724, + "pass@2": 0.4339426114324206, + "pass@4": 0.5158208758197257, + "pass@8": 0.5814749933202288, + "pass@16": 0.6319409424520197, + "pass@32": 0.6743801223415203, + "pass@64": 0.7149931804501817, + "pass@128": 0.7551622418879056 + }, + "17": { + "pass@1": 0.2772260918674699, + "pass@2": 0.3546075295275589, + "pass@4": 0.4294549452026328, + "pass@8": 0.5000140886921921, + "pass@16": 0.5642828001448222, + "pass@32": 0.6196617673206312, + "pass@64": 0.6653444240142372, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.2385836693548387, + "pass@2": 0.3159309912369824, + "pass@4": 0.39028818504541773, + "pass@8": 0.4506015189042406, + "pass@16": 0.4972679372856847, + "pass@32": 0.5350537274315351, + "pass@64": 0.5658234390872118, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.17352719033232628, + "pass@2": 0.24078118681161823, + "pass@4": 0.3101919103012425, + "pass@8": 0.3732346835782246, + "pass@16": 0.4258597022657869, + "pass@32": 0.4729959503972069, + "pass@64": 0.51894019609402, + "pass@128": 0.5589123867069486 + }, + "20": { + "pass@1": 0.165868993993994, + "pass@2": 0.22920915502116282, + "pass@4": 0.2946917814890254, + "pass@8": 0.3602414401773009, + "pass@16": 0.42153930178411225, + "pass@32": 0.4743852103979818, + "pass@64": 0.5184722761047903, + "pass@128": 0.5555555555555556 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7302919407894737, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7302919407894737, + "pass@2": 0.7659925145047483, + "pass@4": 0.7958980826968828, + "pass@8": 0.8207104803510987, + "pass@16": 0.8409706486024597, + "pass@32": 0.8578509037396045, + "pass@64": 0.8725085624408543, + "pass@128": 0.8851578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954296875, + "pass@2": 0.9661904527559058, + "pass@4": 0.9746338377390329, + "pass@8": 0.9803727820809247, + "pass@16": 0.984347680858482, + "pass@32": 0.9872299909032018, + "pass@64": 0.9893549245650947, + "pass@128": 0.991 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997109375, + "pass@2": 0.999918061023622, + "pass@4": 0.9999938090551181, + "pass@8": 0.9999999729957695, + "pass@16": 0.999999999999862, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.995140625, + "pass@2": 0.9963550688976379, + "pass@4": 0.997230504030746, + "pass@8": 0.9979210739667388, + "pass@16": 0.998400288705097, + "pass@32": 0.998829650243074, + "pass@64": 0.9993779525919387, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991328125, + "pass@2": 0.9993145915354331, + "pass@4": 0.9995343371766029, + "pass@8": 0.9997890613208846, + "pass@16": 0.9999604948521619, + "pass@32": 0.9999991088495628, + "pass@64": 0.9999999999731547, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977578125, + "pass@2": 0.9984231053149607, + "pass@4": 0.9988245402137234, + "pass@8": 0.9990286195412216, + "pass@16": 0.9991241570481809, + "pass@32": 0.9992499998305313, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.996796875, + "pass@2": 0.9983270177165355, + "pass@4": 0.9989689832208474, + "pass@8": 0.9991198680433677, + "pass@16": 0.9992352358817035, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943828125, + "pass@2": 0.995630782480315, + "pass@4": 0.9963846165166853, + "pass@8": 0.9970685073427377, + "pass@16": 0.9977582609730773, + "pass@32": 0.9984989827276101, + "pass@64": 0.999238082753677, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9493203125, + "pass@2": 0.9654307332677168, + "pass@4": 0.9747091845706785, + "pass@8": 0.9807236154980903, + "pass@16": 0.9846030327276756, + "pass@32": 0.9876037567845825, + "pass@64": 0.9906123675412761, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.8656640625, + "pass@2": 0.8981660925196859, + "pass@4": 0.9162192402512177, + "pass@8": 0.9294925566724376, + "pass@16": 0.9403418552190832, + "pass@32": 0.9504850673157726, + "pass@64": 0.9600327044081675, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.7539765625, + "pass@2": 0.8214111712598431, + "pass@4": 0.8642939338207711, + "pass@8": 0.8939748081673056, + "pass@16": 0.9152320369752246, + "pass@32": 0.931471224927995, + "pass@64": 0.9453016174913549, + "pass@128": 0.957 + }, + "14": { + "pass@1": 0.66509375, + "pass@2": 0.7435506889763775, + "pass@4": 0.7978263482377215, + "pass@8": 0.8370823461640506, + "pass@16": 0.8670369761030744, + "pass@32": 0.8905706816269232, + "pass@64": 0.909815526205601, + "pass@128": 0.924 + }, + "15": { + "pass@1": 0.5480078125, + "pass@2": 0.6441775344488198, + "pass@4": 0.7138529732845889, + "pass@8": 0.7638165706839227, + "pass@16": 0.8011898814003716, + "pass@32": 0.8304545822003983, + "pass@64": 0.8536875341305782, + "pass@128": 0.873 + }, + "16": { + "pass@1": 0.3315, + "pass@2": 0.4212034940944888, + "pass@4": 0.5018796497937756, + "pass@8": 0.569041091355488, + "pass@16": 0.6232936592313723, + "pass@32": 0.6689611183103249, + "pass@64": 0.7101747586666186, + "pass@128": 0.748 + }, + "17": { + "pass@1": 0.264140625, + "pass@2": 0.34535888287401634, + "pass@4": 0.42405996878515173, + "pass@8": 0.493988943111484, + "pass@16": 0.5533799983646059, + "pass@32": 0.6040125546424108, + "pass@64": 0.6488027969675257, + "pass@128": 0.688 + }, + "18": { + "pass@1": 0.2277890625, + "pass@2": 0.3007801427165356, + "pass@4": 0.37204404443194605, + "pass@8": 0.435300861022257, + "pass@16": 0.48862231997077116, + "pass@32": 0.5342048573239302, + "pass@64": 0.5734251003614904, + "pass@128": 0.609 + }, + "19": { + "pass@1": 0.1720546875, + "pass@2": 0.2371135580708661, + "pass@4": 0.30388390813648286, + "pass@8": 0.3649321499143648, + "pass@16": 0.4171717877808307, + "pass@32": 0.4631558514273401, + "pass@64": 0.506340857897091, + "pass@128": 0.546 + }, + "20": { + "pass@1": 0.16078125, + "pass@2": 0.2225063976377949, + "pass@4": 0.2877236919760029, + "pass@8": 0.35184629878963847, + "pass@16": 0.40874465735495447, + "pass@32": 0.45500076756063307, + "pass@64": 0.49224649431868345, + "pass@128": 0.522 + } + }, + "per_op_accuracy": { + "10": 0.954296875, + "2": 1.0, + "3": 1.0, + "4": 0.9997109375, + "5": 0.995140625, + "6": 0.9991328125, + "7": 0.9977578125, + "8": 0.996796875, + "9": 0.9943828125, + "11": 0.9493203125, + "12": 0.8656640625, + "13": 0.7539765625, + "14": 0.66509375, + "15": 0.5480078125, + "16": 0.3315, + "17": 0.264140625, + "18": 0.2277890625, + "19": 0.1720546875, + "20": 0.16078125 + }, + "per_op_avg_response_len": { + "10": 281.391109375, + "2": 115.720453125, + "3": 144.8661328125, + "4": 160.0339921875, + "5": 173.204265625, + "6": 199.208125, + "7": 224.3385, + "8": 243.8726953125, + "9": 259.81234375, + "11": 298.9996796875, + "12": 308.2072109375, + "13": 324.847875, + "14": 344.7609296875, + "15": 353.43428125, + "16": 337.8661484375, + "17": 352.958828125, + "18": 356.199546875, + "19": 343.0367578125, + "20": 353.50103125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.391109375, + "2": 115.720453125, + "3": 144.8661328125, + "4": 160.0339921875, + "5": 173.204265625, + "6": 199.208125, + "7": 224.3385, + "8": 243.8726953125, + "9": 259.81234375, + "11": 298.9996796875, + "12": 308.2072109375, + "13": 324.847875, + "14": 344.7609296875, + "15": 353.43428125, + "16": 337.8661484375, + "17": 352.958828125, + "18": 356.199546875, + "19": 343.0367578125, + "20": 353.50103125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 589119, + "answer_accuracy": 0.7275517210717674, + "avg_response_len": 259.71025331963324, + "resp_tokens_sum": 210294664, + "pass_at_k": { + "pass@1": 0.7275517210717674, + "pass@2": 0.7635894180310674, + "pass@4": 0.7937236054867095, + "pass@8": 0.8186666406471939, + "pass@16": 0.8389692396158039, + "pass@32": 0.8557389237874758, + "pass@64": 0.8703361317609539, + "pass@128": 0.8830224470439456 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.963472529498525, + "pass@2": 0.9744087229693631, + "pass@4": 0.9803487268737424, + "pass@8": 0.9836354699131256, + "pass@16": 0.9861514017924913, + "pass@32": 0.9888094834427503, + "pass@64": 0.9916772440992451, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990966796875, + "pass@2": 0.9997439406988189, + "pass@4": 0.9999806532972441, + "pass@8": 0.9999999156117797, + "pass@16": 0.9999999999995689, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951388888888889, + "pass@2": 0.9965351987251593, + "pass@4": 0.9979581129739735, + "pass@8": 0.9991604105838077, + "pass@16": 0.999826156800224, + "pass@32": 0.9999942644471409, + "pass@64": 0.9999999994794706, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974270209580839, + "pass@2": 0.997947878848602, + "pass@4": 0.9986057999299488, + "pass@8": 0.9993684470685168, + "pass@16": 0.9998817211142573, + "pass@32": 0.9999973318849184, + "pass@64": 0.9999999999196246, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976423816568047, + "pass@2": 0.9986568455015609, + "pass@4": 0.9994379571821275, + "pass@8": 0.9999004126844195, + "pass@16": 0.999997506107353, + "pass@32": 0.9999999994986136, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9992487980769231, + "pass@2": 0.9998986567484354, + "pass@4": 0.9999973395753415, + "pass@8": 0.9999999989008069, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953125, + "pass@2": 0.996617500277254, + "pass@4": 0.9974524414201746, + "pass@8": 0.9980863720132519, + "pass@16": 0.9987573795437611, + "pass@32": 0.9995185120065893, + "pass@64": 0.999961060003192, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9400230169971672, + "pass@2": 0.9581030704200219, + "pass@4": 0.9695777809997548, + "pass@8": 0.9776203275632077, + "pass@16": 0.9829647487484047, + "pass@32": 0.9867373748424951, + "pass@64": 0.990728249665513, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.859816384180791, + "pass@2": 0.8918441934694605, + "pass@4": 0.9098151451937151, + "pass@8": 0.9241309150223803, + "pass@16": 0.9363586934573225, + "pass@32": 0.9478360071414645, + "pass@64": 0.9589968897614609, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7661637931034483, + "pass@2": 0.8327435020857502, + "pass@4": 0.8767341852644595, + "pass@8": 0.9089967422970507, + "pass@16": 0.9321636362076665, + "pass@32": 0.94779652546968, + "pass@64": 0.9595366262742799, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6715342679127726, + "pass@2": 0.7542328967056693, + "pass@4": 0.8122993729171699, + "pass@8": 0.8509755820397881, + "pass@16": 0.8762950400746685, + "pass@32": 0.8949873380135102, + "pass@64": 0.9119272070496415, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5707640895953757, + "pass@2": 0.6658713474580124, + "pass@4": 0.7362232691469921, + "pass@8": 0.7895324052420507, + "pass@16": 0.829999658967477, + "pass@32": 0.8602938952993778, + "pass@64": 0.8807124344185117, + "pass@128": 0.8930635838150289 + }, + "16": { + "pass@1": 0.3118951612903226, + "pass@2": 0.40475576263652524, + "pass@4": 0.4862871569481233, + "pass@8": 0.554336074823824, + "pass@16": 0.6110271182506712, + "pass@32": 0.6587839935388526, + "pass@64": 0.7001171727816106, + "pass@128": 0.7387096774193549 + }, + "17": { + "pass@1": 0.244829299132948, + "pass@2": 0.328066328910837, + "pass@4": 0.41020303468208086, + "pass@8": 0.48089300913906174, + "pass@16": 0.5372670405574277, + "pass@32": 0.5825153543250646, + "pass@64": 0.621207883664678, + "pass@128": 0.653179190751445 + }, + "18": { + "pass@1": 0.20457589285714287, + "pass@2": 0.2681696428571427, + "pass@4": 0.3305764103594193, + "pass@8": 0.3901694784094153, + "pass@16": 0.44529528231015914, + "pass@32": 0.494447129452626, + "pass@64": 0.5396755944182013, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.17171077327327328, + "pass@2": 0.24016523905795556, + "pass@4": 0.30821597384786364, + "pass@8": 0.3693761252170798, + "pass@16": 0.42273905856526567, + "pass@32": 0.47019284079490814, + "pass@64": 0.5157973767607983, + "pass@128": 0.5615615615615616 + }, + "20": { + "pass@1": 0.1664085960960961, + "pass@2": 0.2306275271334326, + "pass@4": 0.2956622741977073, + "pass@8": 0.355173554615156, + "pass@16": 0.4063776034385708, + "pass@32": 0.4498967766195675, + "pass@64": 0.4870903338771562, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 599675, + "answer_accuracy": 0.7303134742790335, + "avg_response_len": 268.7120822778644, + "resp_tokens_sum": 220644865, + "pass_at_k": { + "pass@1": 0.7303134742790335, + "pass@2": 0.7658481635990868, + "pass@4": 0.7958019730191505, + "pass@8": 0.8207241724801014, + "pass@16": 0.8408275202187017, + "pass@32": 0.8574976101121811, + "pass@64": 0.8721876179734757, + "pass@128": 0.8849571317225253 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9395609662576687, + "pass@2": 0.9548636690256509, + "pass@4": 0.966663017487078, + "pass@8": 0.9743895360249896, + "pass@16": 0.9788663525185067, + "pass@32": 0.981713951963166, + "pass@64": 0.9837144251220994, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9971693840579711, + "pass@2": 0.9972362490014836, + "pass@4": 0.9973668033343658, + "pass@8": 0.997615447525581, + "pass@16": 0.9980647799459851, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999776146131805, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9971324060693642, + "pass@2": 0.997154985549133, + "pass@4": 0.9972001445086706, + "pass@8": 0.9972904624277457, + "pass@16": 0.997471098265896, + "pass@32": 0.9978323699421965, + "pass@64": 0.9985549132947977, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959947183098592, + "pass@2": 0.9968451951868693, + "pass@4": 0.997296264463421, + "pass@8": 0.9975244495786237, + "pass@16": 0.9978457356837299, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9941401645527842, + "pass@4": 0.994290434878275, + "pass@8": 0.994563039275777, + "pass@16": 0.9950905276062049, + "pass@32": 0.9960747795747089, + "pass@64": 0.9977603847423264, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9549134036144579, + "pass@2": 0.9673173501090979, + "pass@4": 0.9744871380911725, + "pass@8": 0.979511684978502, + "pass@16": 0.9829385174486236, + "pass@32": 0.9856195234202649, + "pass@64": 0.9875988046336269, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8561422413793104, + "pass@2": 0.8865071088292648, + "pass@4": 0.9036359111067229, + "pass@8": 0.9166247203980661, + "pass@16": 0.9272387717823719, + "pass@32": 0.9379651890883318, + "pass@64": 0.9487072364211467, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.7606573879551821, + "pass@2": 0.8298419269282518, + "pass@4": 0.8707777709508995, + "pass@8": 0.896675537557301, + "pass@16": 0.9137837552594845, + "pass@32": 0.9273137994362725, + "pass@64": 0.9404390148244085, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.6662097953216374, + "pass@2": 0.7445089100704514, + "pass@4": 0.7975902315938572, + "pass@8": 0.8359411040303549, + "pass@16": 0.8656772291606879, + "pass@32": 0.8890270125392543, + "pass@64": 0.9079604648318258, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5242897727272727, + "pass@2": 0.6281238815318541, + "pass@4": 0.7041429557100817, + "pass@8": 0.7557992599136547, + "pass@16": 0.792250838503205, + "pass@32": 0.8210298940378924, + "pass@64": 0.848988614620591, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.33751780626780625, + "pass@2": 0.42342638860847526, + "pass@4": 0.5021861944073225, + "pass@8": 0.5700196165941456, + "pass@16": 0.6257757067875516, + "pass@32": 0.6727157231894736, + "pass@64": 0.7144038374122713, + "pass@128": 0.7492877492877493 + }, + "17": { + "pass@1": 0.27139945652173914, + "pass@2": 0.3544044511419766, + "pass@4": 0.43338719558347116, + "pass@8": 0.5018487096383886, + "pass@16": 0.5594525238622866, + "pass@32": 0.6109768176878541, + "pass@64": 0.6613991318224222, + "pass@128": 0.7080745341614907 + }, + "18": { + "pass@1": 0.24184283088235295, + "pass@2": 0.32053576597962, + "pass@4": 0.39809695130020534, + "pass@8": 0.46780903740778607, + "pass@16": 0.5253409135401563, + "pass@32": 0.5743579603286321, + "pass@64": 0.6150981652884812, + "pass@128": 0.6470588235294118 + }, + "19": { + "pass@1": 0.17094494047619047, + "pass@2": 0.23047607330333703, + "pass@4": 0.2933763884983127, + "pass@8": 0.35234886890680117, + "pass@16": 0.40309559472804163, + "pass@32": 0.44648802339630994, + "pass@64": 0.4845569241923335, + "pass@128": 0.5178571428571429 + }, + "20": { + "pass@1": 0.15009824101796407, + "pass@2": 0.2077268936536376, + "pass@4": 0.27286165099497306, + "pass@8": 0.3401589985741033, + "pass@16": 0.39834828717305787, + "pass@32": 0.440763071508351, + "pass@64": 0.4712399556727392, + "pass@128": 0.4940119760479042 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587276, + "answer_accuracy": 0.733039423230548, + "avg_response_len": 289.11085412006713, + "resp_tokens_sum": 231621739, + "pass_at_k": { + "pass@1": 0.733039423230548, + "pass@2": 0.7685692838532925, + "pass@4": 0.7981943419099254, + "pass@8": 0.8227621651184949, + "pass@16": 0.8431401775737536, + "pass@32": 0.8603475907178547, + "pass@64": 0.8750331918128493, + "pass@128": 0.8875219683655536 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9593516791044776, + "pass@2": 0.9688965360206839, + "pass@4": 0.9766073899344672, + "pass@8": 0.9828936389141163, + "pass@16": 0.9878564917307295, + "pass@32": 0.9909994856003504, + "pass@64": 0.9924938394795402, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9930836397058823, + "pass@2": 0.9952940452755904, + "pass@4": 0.9964180920399656, + "pass@8": 0.9970829859897469, + "pass@16": 0.9974197065637099, + "pass@32": 0.9977941128663109, + "pass@64": 0.9985294117647059, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985660601265823, + "pass@2": 0.9995616029602312, + "pass@4": 0.9999470274878615, + "pass@8": 0.9999993039679993, + "pass@16": 0.9999999999490367, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953547297297297, + "pass@2": 0.9984342117708261, + "pass@4": 0.9997887068733525, + "pass@8": 0.9999960624201342, + "pass@16": 0.9999999991410792, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941971463022508, + "pass@2": 0.9961053212902246, + "pass@4": 0.9974147735150468, + "pass@8": 0.9985973960126167, + "pass@16": 0.9994828135516711, + "pass@32": 0.9999386964222448, + "pass@64": 0.9999998004778354, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.953844246031746, + "pass@2": 0.9716539338832647, + "pass@4": 0.980693644544432, + "pass@8": 0.9854786046203673, + "pass@16": 0.9881932972271286, + "pass@32": 0.990665973617949, + "pass@64": 0.9936587056221776, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.8812834480122325, + "pass@2": 0.9163837318500325, + "pass@4": 0.9354275939131463, + "pass@8": 0.9478499295031567, + "pass@16": 0.9574364206012684, + "pass@32": 0.9655664387416397, + "pass@64": 0.9722024373524911, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7346161265432098, + "pass@2": 0.8009642783367357, + "pass@4": 0.8449014336170942, + "pass@8": 0.8762088903289816, + "pass@16": 0.9001575197448859, + "pass@32": 0.919978725013635, + "pass@64": 0.9366441525232889, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6578264094955489, + "pass@2": 0.7324032103553814, + "pass@4": 0.7842800899887509, + "pass@8": 0.8250069043053341, + "pass@16": 0.859598385359498, + "pass@32": 0.8879302903150209, + "pass@64": 0.9096866877453451, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.5478636188271605, + "pass@2": 0.6373616275396131, + "pass@4": 0.6998535394302561, + "pass@8": 0.7445203786995883, + "pass@16": 0.7795284650974312, + "pass@32": 0.8081883623281141, + "pass@64": 0.8296135462869662, + "pass@128": 0.845679012345679 + }, + "16": { + "pass@1": 0.34319690265486724, + "pass@2": 0.4339426114324206, + "pass@4": 0.5158208758197257, + "pass@8": 0.5814749933202288, + "pass@16": 0.6319409424520197, + "pass@32": 0.6743801223415203, + "pass@64": 0.7149931804501817, + "pass@128": 0.7551622418879056 + }, + "17": { + "pass@1": 0.2772260918674699, + "pass@2": 0.3546075295275589, + "pass@4": 0.4294549452026328, + "pass@8": 0.5000140886921921, + "pass@16": 0.5642828001448222, + "pass@32": 0.6196617673206312, + "pass@64": 0.6653444240142372, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.2385836693548387, + "pass@2": 0.3159309912369824, + "pass@4": 0.39028818504541773, + "pass@8": 0.4506015189042406, + "pass@16": 0.4972679372856847, + "pass@32": 0.5350537274315351, + "pass@64": 0.5658234390872118, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.17352719033232628, + "pass@2": 0.24078118681161823, + "pass@4": 0.3101919103012425, + "pass@8": 0.3732346835782246, + "pass@16": 0.4258597022657869, + "pass@32": 0.4729959503972069, + "pass@64": 0.51894019609402, + "pass@128": 0.5589123867069486 + }, + "20": { + "pass@1": 0.165868993993994, + "pass@2": 0.22920915502116282, + "pass@4": 0.2946917814890254, + "pass@8": 0.3602414401773009, + "pass@16": 0.42153930178411225, + "pass@32": 0.4743852103979818, + "pass@64": 0.5184722761047903, + "pass@128": 0.5555555555555556 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b07787d374aa442f591c4631562c7f9e9b31b9b7 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a76036c4b5ea7237600f633a5ce554d220979a27bb6ab9d6a27597b3a3afaf50 +size 2597204819 diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/summary.csv b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..0bea3f4652f348e9d5d89b208e5f718832dc3338 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op7-10,0.9930243055555555,0.4938328125,0.7302919407894737,0.0,0.0,0.0,200.27195746527778,337.38122890625,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/summary.json b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7590aef33708ff11cc8dd9f067a8eb5c6df3efa7 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op7-10", + "id_acc": 0.9930243055555555, + "ood_acc": 0.4938328125, + "total_acc": 0.7302919407894737, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.27195746527778, + "ood_avg_resp_len": 337.38122890625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 589119, + "answer_accuracy": 0.7275517210717674, + "avg_response_len": 259.71025331963324, + "resp_tokens_sum": 210294664, + "pass_at_k": { + "pass@1": 0.7275517210717674, + "pass@2": 0.7635894180310674, + "pass@4": 0.7937236054867095, + "pass@8": 0.8186666406471939, + "pass@16": 0.8389692396158039, + "pass@32": 0.8557389237874758, + "pass@64": 0.8703361317609539, + "pass@128": 0.8830224470439456 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.963472529498525, + "pass@2": 0.9744087229693631, + "pass@4": 0.9803487268737424, + "pass@8": 0.9836354699131256, + "pass@16": 0.9861514017924913, + "pass@32": 0.9888094834427503, + "pass@64": 0.9916772440992451, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990966796875, + "pass@2": 0.9997439406988189, + "pass@4": 0.9999806532972441, + "pass@8": 0.9999999156117797, + "pass@16": 0.9999999999995689, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951388888888889, + "pass@2": 0.9965351987251593, + "pass@4": 0.9979581129739735, + "pass@8": 0.9991604105838077, + "pass@16": 0.999826156800224, + "pass@32": 0.9999942644471409, + "pass@64": 0.9999999994794706, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974270209580839, + "pass@2": 0.997947878848602, + "pass@4": 0.9986057999299488, + "pass@8": 0.9993684470685168, + "pass@16": 0.9998817211142573, + "pass@32": 0.9999973318849184, + "pass@64": 0.9999999999196246, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976423816568047, + "pass@2": 0.9986568455015609, + "pass@4": 0.9994379571821275, + "pass@8": 0.9999004126844195, + "pass@16": 0.999997506107353, + "pass@32": 0.9999999994986136, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9992487980769231, + "pass@2": 0.9998986567484354, + "pass@4": 0.9999973395753415, + "pass@8": 0.9999999989008069, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953125, + "pass@2": 0.996617500277254, + "pass@4": 0.9974524414201746, + "pass@8": 0.9980863720132519, + "pass@16": 0.9987573795437611, + "pass@32": 0.9995185120065893, + "pass@64": 0.999961060003192, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9400230169971672, + "pass@2": 0.9581030704200219, + "pass@4": 0.9695777809997548, + "pass@8": 0.9776203275632077, + "pass@16": 0.9829647487484047, + "pass@32": 0.9867373748424951, + "pass@64": 0.990728249665513, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.859816384180791, + "pass@2": 0.8918441934694605, + "pass@4": 0.9098151451937151, + "pass@8": 0.9241309150223803, + "pass@16": 0.9363586934573225, + "pass@32": 0.9478360071414645, + "pass@64": 0.9589968897614609, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7661637931034483, + "pass@2": 0.8327435020857502, + "pass@4": 0.8767341852644595, + "pass@8": 0.9089967422970507, + "pass@16": 0.9321636362076665, + "pass@32": 0.94779652546968, + "pass@64": 0.9595366262742799, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6715342679127726, + "pass@2": 0.7542328967056693, + "pass@4": 0.8122993729171699, + "pass@8": 0.8509755820397881, + "pass@16": 0.8762950400746685, + "pass@32": 0.8949873380135102, + "pass@64": 0.9119272070496415, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5707640895953757, + "pass@2": 0.6658713474580124, + "pass@4": 0.7362232691469921, + "pass@8": 0.7895324052420507, + "pass@16": 0.829999658967477, + "pass@32": 0.8602938952993778, + "pass@64": 0.8807124344185117, + "pass@128": 0.8930635838150289 + }, + "16": { + "pass@1": 0.3118951612903226, + "pass@2": 0.40475576263652524, + "pass@4": 0.4862871569481233, + "pass@8": 0.554336074823824, + "pass@16": 0.6110271182506712, + "pass@32": 0.6587839935388526, + "pass@64": 0.7001171727816106, + "pass@128": 0.7387096774193549 + }, + "17": { + "pass@1": 0.244829299132948, + "pass@2": 0.328066328910837, + "pass@4": 0.41020303468208086, + "pass@8": 0.48089300913906174, + "pass@16": 0.5372670405574277, + "pass@32": 0.5825153543250646, + "pass@64": 0.621207883664678, + "pass@128": 0.653179190751445 + }, + "18": { + "pass@1": 0.20457589285714287, + "pass@2": 0.2681696428571427, + "pass@4": 0.3305764103594193, + "pass@8": 0.3901694784094153, + "pass@16": 0.44529528231015914, + "pass@32": 0.494447129452626, + "pass@64": 0.5396755944182013, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.17171077327327328, + "pass@2": 0.24016523905795556, + "pass@4": 0.30821597384786364, + "pass@8": 0.3693761252170798, + "pass@16": 0.42273905856526567, + "pass@32": 0.47019284079490814, + "pass@64": 0.5157973767607983, + "pass@128": 0.5615615615615616 + }, + "20": { + "pass@1": 0.1664085960960961, + "pass@2": 0.2306275271334326, + "pass@4": 0.2956622741977073, + "pass@8": 0.355173554615156, + "pass@16": 0.4063776034385708, + "pass@32": 0.4498967766195675, + "pass@64": 0.4870903338771562, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 599675, + "answer_accuracy": 0.7303134742790335, + "avg_response_len": 268.7120822778644, + "resp_tokens_sum": 220644865, + "pass_at_k": { + "pass@1": 0.7303134742790335, + "pass@2": 0.7658481635990868, + "pass@4": 0.7958019730191505, + "pass@8": 0.8207241724801014, + "pass@16": 0.8408275202187017, + "pass@32": 0.8574976101121811, + "pass@64": 0.8721876179734757, + "pass@128": 0.8849571317225253 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9395609662576687, + "pass@2": 0.9548636690256509, + "pass@4": 0.966663017487078, + "pass@8": 0.9743895360249896, + "pass@16": 0.9788663525185067, + "pass@32": 0.981713951963166, + "pass@64": 0.9837144251220994, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9971693840579711, + "pass@2": 0.9972362490014836, + "pass@4": 0.9973668033343658, + "pass@8": 0.997615447525581, + "pass@16": 0.9980647799459851, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999776146131805, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9971324060693642, + "pass@2": 0.997154985549133, + "pass@4": 0.9972001445086706, + "pass@8": 0.9972904624277457, + "pass@16": 0.997471098265896, + "pass@32": 0.9978323699421965, + "pass@64": 0.9985549132947977, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959947183098592, + "pass@2": 0.9968451951868693, + "pass@4": 0.997296264463421, + "pass@8": 0.9975244495786237, + "pass@16": 0.9978457356837299, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9941401645527842, + "pass@4": 0.994290434878275, + "pass@8": 0.994563039275777, + "pass@16": 0.9950905276062049, + "pass@32": 0.9960747795747089, + "pass@64": 0.9977603847423264, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9549134036144579, + "pass@2": 0.9673173501090979, + "pass@4": 0.9744871380911725, + "pass@8": 0.979511684978502, + "pass@16": 0.9829385174486236, + "pass@32": 0.9856195234202649, + "pass@64": 0.9875988046336269, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8561422413793104, + "pass@2": 0.8865071088292648, + "pass@4": 0.9036359111067229, + "pass@8": 0.9166247203980661, + "pass@16": 0.9272387717823719, + "pass@32": 0.9379651890883318, + "pass@64": 0.9487072364211467, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.7606573879551821, + "pass@2": 0.8298419269282518, + "pass@4": 0.8707777709508995, + "pass@8": 0.896675537557301, + "pass@16": 0.9137837552594845, + "pass@32": 0.9273137994362725, + "pass@64": 0.9404390148244085, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.6662097953216374, + "pass@2": 0.7445089100704514, + "pass@4": 0.7975902315938572, + "pass@8": 0.8359411040303549, + "pass@16": 0.8656772291606879, + "pass@32": 0.8890270125392543, + "pass@64": 0.9079604648318258, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5242897727272727, + "pass@2": 0.6281238815318541, + "pass@4": 0.7041429557100817, + "pass@8": 0.7557992599136547, + "pass@16": 0.792250838503205, + "pass@32": 0.8210298940378924, + "pass@64": 0.848988614620591, + "pass@128": 0.8787878787878788 + }, + "16": { + "pass@1": 0.33751780626780625, + "pass@2": 0.42342638860847526, + "pass@4": 0.5021861944073225, + "pass@8": 0.5700196165941456, + "pass@16": 0.6257757067875516, + "pass@32": 0.6727157231894736, + "pass@64": 0.7144038374122713, + "pass@128": 0.7492877492877493 + }, + "17": { + "pass@1": 0.27139945652173914, + "pass@2": 0.3544044511419766, + "pass@4": 0.43338719558347116, + "pass@8": 0.5018487096383886, + "pass@16": 0.5594525238622866, + "pass@32": 0.6109768176878541, + "pass@64": 0.6613991318224222, + "pass@128": 0.7080745341614907 + }, + "18": { + "pass@1": 0.24184283088235295, + "pass@2": 0.32053576597962, + "pass@4": 0.39809695130020534, + "pass@8": 0.46780903740778607, + "pass@16": 0.5253409135401563, + "pass@32": 0.5743579603286321, + "pass@64": 0.6150981652884812, + "pass@128": 0.6470588235294118 + }, + "19": { + "pass@1": 0.17094494047619047, + "pass@2": 0.23047607330333703, + "pass@4": 0.2933763884983127, + "pass@8": 0.35234886890680117, + "pass@16": 0.40309559472804163, + "pass@32": 0.44648802339630994, + "pass@64": 0.4845569241923335, + "pass@128": 0.5178571428571429 + }, + "20": { + "pass@1": 0.15009824101796407, + "pass@2": 0.2077268936536376, + "pass@4": 0.27286165099497306, + "pass@8": 0.3401589985741033, + "pass@16": 0.39834828717305787, + "pass@32": 0.440763071508351, + "pass@64": 0.4712399556727392, + "pass@128": 0.4940119760479042 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587276, + "answer_accuracy": 0.733039423230548, + "avg_response_len": 289.11085412006713, + "resp_tokens_sum": 231621739, + "pass_at_k": { + "pass@1": 0.733039423230548, + "pass@2": 0.7685692838532925, + "pass@4": 0.7981943419099254, + "pass@8": 0.8227621651184949, + "pass@16": 0.8431401775737536, + "pass@32": 0.8603475907178547, + "pass@64": 0.8750331918128493, + "pass@128": 0.8875219683655536 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9593516791044776, + "pass@2": 0.9688965360206839, + "pass@4": 0.9766073899344672, + "pass@8": 0.9828936389141163, + "pass@16": 0.9878564917307295, + "pass@32": 0.9909994856003504, + "pass@64": 0.9924938394795402, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9930836397058823, + "pass@2": 0.9952940452755904, + "pass@4": 0.9964180920399656, + "pass@8": 0.9970829859897469, + "pass@16": 0.9974197065637099, + "pass@32": 0.9977941128663109, + "pass@64": 0.9985294117647059, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985660601265823, + "pass@2": 0.9995616029602312, + "pass@4": 0.9999470274878615, + "pass@8": 0.9999993039679993, + "pass@16": 0.9999999999490367, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953547297297297, + "pass@2": 0.9984342117708261, + "pass@4": 0.9997887068733525, + "pass@8": 0.9999960624201342, + "pass@16": 0.9999999991410792, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941971463022508, + "pass@2": 0.9961053212902246, + "pass@4": 0.9974147735150468, + "pass@8": 0.9985973960126167, + "pass@16": 0.9994828135516711, + "pass@32": 0.9999386964222448, + "pass@64": 0.9999998004778354, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.953844246031746, + "pass@2": 0.9716539338832647, + "pass@4": 0.980693644544432, + "pass@8": 0.9854786046203673, + "pass@16": 0.9881932972271286, + "pass@32": 0.990665973617949, + "pass@64": 0.9936587056221776, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.8812834480122325, + "pass@2": 0.9163837318500325, + "pass@4": 0.9354275939131463, + "pass@8": 0.9478499295031567, + "pass@16": 0.9574364206012684, + "pass@32": 0.9655664387416397, + "pass@64": 0.9722024373524911, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7346161265432098, + "pass@2": 0.8009642783367357, + "pass@4": 0.8449014336170942, + "pass@8": 0.8762088903289816, + "pass@16": 0.9001575197448859, + "pass@32": 0.919978725013635, + "pass@64": 0.9366441525232889, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6578264094955489, + "pass@2": 0.7324032103553814, + "pass@4": 0.7842800899887509, + "pass@8": 0.8250069043053341, + "pass@16": 0.859598385359498, + "pass@32": 0.8879302903150209, + "pass@64": 0.9096866877453451, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.5478636188271605, + "pass@2": 0.6373616275396131, + "pass@4": 0.6998535394302561, + "pass@8": 0.7445203786995883, + "pass@16": 0.7795284650974312, + "pass@32": 0.8081883623281141, + "pass@64": 0.8296135462869662, + "pass@128": 0.845679012345679 + }, + "16": { + "pass@1": 0.34319690265486724, + "pass@2": 0.4339426114324206, + "pass@4": 0.5158208758197257, + "pass@8": 0.5814749933202288, + "pass@16": 0.6319409424520197, + "pass@32": 0.6743801223415203, + "pass@64": 0.7149931804501817, + "pass@128": 0.7551622418879056 + }, + "17": { + "pass@1": 0.2772260918674699, + "pass@2": 0.3546075295275589, + "pass@4": 0.4294549452026328, + "pass@8": 0.5000140886921921, + "pass@16": 0.5642828001448222, + "pass@32": 0.6196617673206312, + "pass@64": 0.6653444240142372, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.2385836693548387, + "pass@2": 0.3159309912369824, + "pass@4": 0.39028818504541773, + "pass@8": 0.4506015189042406, + "pass@16": 0.4972679372856847, + "pass@32": 0.5350537274315351, + "pass@64": 0.5658234390872118, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.17352719033232628, + "pass@2": 0.24078118681161823, + "pass@4": 0.3101919103012425, + "pass@8": 0.3732346835782246, + "pass@16": 0.4258597022657869, + "pass@32": 0.4729959503972069, + "pass@64": 0.51894019609402, + "pass@128": 0.5589123867069486 + }, + "20": { + "pass@1": 0.165868993993994, + "pass@2": 0.22920915502116282, + "pass@4": 0.2946917814890254, + "pass@8": 0.3602414401773009, + "pass@16": 0.42153930178411225, + "pass@32": 0.4743852103979818, + "pass@64": 0.5184722761047903, + "pass@128": 0.5555555555555556 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..eb8840cec7d0ef9b1e423b55e8204d89799f9a92 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0173a351b3890c1e66f8fc44efd2a7b16151592440fb5e41bc67d594332f74ef +size 1789457520 diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a8a2013576bcf7903cb309d62dd61628d3b7a512 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9925381944444445, + "avg_response_len": 199.54979947916667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9925381944444445, + "pass@2": 0.9946197916666658, + "pass@4": 0.9960532251906017, + "pass@8": 0.9970113894344446, + "pass@16": 0.9977232440991999, + "pass@32": 0.9983049478962568, + "pass@64": 0.9987103453068888, + "pass@128": 0.9988888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.95275, + "pass@2": 0.9646134350393707, + "pass@4": 0.9731705104986875, + "pass@8": 0.9791324008089147, + "pass@16": 0.9838400852148089, + "pass@32": 0.9877436014233196, + "pass@64": 0.9902763476517468, + "pass@128": 0.991 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999375, + "pass@2": 0.9999984005905512, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99934375, + "pass@2": 0.9996381643700788, + "pass@4": 0.9998731449193852, + "pass@8": 0.9999852822033031, + "pass@16": 0.9999998535430754, + "pass@32": 0.9999999999968788, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.994640625, + "pass@2": 0.9965073818897637, + "pass@4": 0.997699450974878, + "pass@8": 0.9982503088775315, + "pass@16": 0.998566901237073, + "pass@32": 0.9990204348783889, + "pass@64": 0.9996299212598426, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989453125, + "pass@2": 0.9992125984251969, + "pass@4": 0.9993964112298462, + "pass@8": 0.9996420801536683, + "pass@16": 0.9998812336914318, + "pass@32": 0.9999901022858957, + "pass@64": 0.999999987930507, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9975, + "pass@2": 0.9983570374015749, + "pass@4": 0.9989147164416947, + "pass@8": 0.9993051158777021, + "pass@16": 0.9996133627852174, + "pass@32": 0.9998739015543171, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951796875, + "pass@2": 0.9972514763779529, + "pass@4": 0.9985841584176979, + "pass@8": 0.9992542527706686, + "pass@16": 0.9996094514666293, + "pass@32": 0.9998738852946448, + "pass@64": 0.9999934280775866, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.994546875, + "pass@2": 0.995999630905512, + "pass@4": 0.9968406347019122, + "pass@8": 0.9975330642182132, + "pass@16": 0.9979983089545647, + "pass@32": 0.9982426056328664, + "pass@64": 0.9984999947647286, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.95275, + "2": 1.0, + "3": 0.9999375, + "4": 0.99934375, + "5": 0.994640625, + "6": 0.9989453125, + "7": 0.9975, + "8": 0.9951796875, + "9": 0.994546875 + }, + "per_op_avg_response_len": { + "10": 280.125859375, + "2": 115.466796875, + "3": 144.4380546875, + "4": 159.5991484375, + "5": 172.673078125, + "6": 198.4890546875, + "7": 223.461984375, + "8": 242.948734375, + "9": 258.745484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.125859375, + "2": 115.466796875, + "3": 144.4380546875, + "4": 159.5991484375, + "5": 172.673078125, + "6": 198.4890546875, + "7": 223.461984375, + "8": 242.948734375, + "9": 258.745484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376348, + "answer_accuracy": 0.9929816784869976, + "avg_response_len": 190.07023862293144, + "resp_tokens_sum": 72038141, + "pass_at_k": { + "pass@1": 0.9929816784869976, + "pass@2": 0.9952723246695229, + "pass@4": 0.9967723432683883, + "pass@8": 0.9976749791749441, + "pass@16": 0.9983149296696602, + "pass@32": 0.9988513610944374, + "pass@64": 0.9992337622147628, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.958218104719764, + "pass@2": 0.9703566679441621, + "pass@4": 0.977850502647346, + "pass@8": 0.9821515252532051, + "pass@16": 0.9859468847525427, + "pass@32": 0.990017335559609, + "pass@64": 0.9933073369353381, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981201171875, + "pass@2": 0.9988750307578741, + "pass@4": 0.9996035822670603, + "pass@8": 0.9999540068853223, + "pass@16": 0.999999542322111, + "pass@32": 0.9999999999902464, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9953125, + "pass@2": 0.997645997375328, + "pass@4": 0.999239501312336, + "pass@8": 0.999901179379142, + "pass@16": 0.9999982194545234, + "pass@32": 0.999999999792329, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9976487723372154, + "pass@4": 0.9981928522033549, + "pass@8": 0.9989283836936177, + "pass@16": 0.9996444122497958, + "pass@32": 0.9999703661254363, + "pass@64": 0.9999999638637934, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9970876479289941, + "pass@2": 0.9982957589805712, + "pass@4": 0.9992151179882099, + "pass@8": 0.9998073596919187, + "pass@16": 0.9999900064095701, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9971454326923077, + "pass@2": 0.9990662224914194, + "pass@4": 0.9998395386874719, + "pass@8": 0.9999942233437229, + "pass@16": 0.9999999948628073, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953125, + "pass@2": 0.9966337889541976, + "pass@4": 0.9978505357604947, + "pass@8": 0.998977259593415, + "pass@16": 0.9997108898434905, + "pass@32": 0.999980014289343, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387493, + "answer_accuracy": 0.9919033625491481, + "avg_response_len": 197.5464475139253, + "resp_tokens_sum": 77172705, + "pass_at_k": { + "pass@1": 0.9919033625491481, + "pass@2": 0.9936771405222858, + "pass@4": 0.9948725207882763, + "pass@8": 0.9957642612403389, + "pass@16": 0.9965251216152463, + "pass@32": 0.9971934304500014, + "pass@64": 0.9977037031041751, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.94140625, + "pass@2": 0.9557011134727794, + "pass@4": 0.9657509209930045, + "pass@8": 0.9726733106639606, + "pass@16": 0.9776191432638202, + "pass@32": 0.9811505532784799, + "pass@64": 0.9835180765731945, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998816287878788, + "pass@2": 0.9999962717728466, + "pass@4": 0.9999999985797231, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9970108695652173, + "pass@2": 0.99722876012781, + "pass@4": 0.9973667938246848, + "pass@8": 0.997615447525581, + "pass@16": 0.9980647799459851, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998209169054442, + "pass@2": 0.999994359587573, + "pass@4": 0.9999999959711339, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972001445086706, + "pass@2": 0.9974181164944701, + "pass@4": 0.9976984244382747, + "pass@8": 0.9981811378621208, + "pass@16": 0.9988923141492616, + "pass@32": 0.999635564735464, + "pass@64": 0.9999810060045892, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9962367957746479, + "pass@2": 0.9971404707774204, + "pass@4": 0.9977174318879153, + "pass@8": 0.9982268251618137, + "pass@16": 0.9989203963062453, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935207709580839, + "pass@2": 0.994047338394078, + "pass@4": 0.9941055330284314, + "pass@8": 0.9941991017964071, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379563, + "answer_accuracy": 0.9927472171074657, + "avg_response_len": 210.99379341730832, + "resp_tokens_sum": 80670523, + "pass_at_k": { + "pass@1": 0.9927472171074657, + "pass@2": 0.9949361027048972, + "pass@4": 0.9965467642624349, + "pass@8": 0.9976278427410368, + "pass@16": 0.9983609035725498, + "pass@32": 0.9988989960268758, + "pass@64": 0.9992200321292747, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9582555970149254, + "pass@2": 0.967474512281114, + "pass@4": 0.975654895097068, + "pass@8": 0.9823627894675429, + "pass@16": 0.987761942028932, + "pass@32": 0.9918586397009777, + "pass@64": 0.9937858730382253, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998629385964912, + "pass@2": 0.9999946038817517, + "pass@4": 0.9999999958886717, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9916130514705882, + "pass@2": 0.994720501389532, + "pass@4": 0.9966101887999295, + "pass@8": 0.9973650234611066, + "pass@16": 0.9977503324339216, + "pass@32": 0.9983499305180185, + "pass@64": 0.9992704955998147, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982693829113924, + "pass@2": 0.9994506410096681, + "pass@4": 0.9999251636266987, + "pass@8": 0.9999985905112019, + "pass@16": 0.9999999997567663, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.992210960960961, + "pass@2": 0.995669513016954, + "pass@4": 0.9983319340150049, + "pass@8": 0.9996562527176672, + "pass@16": 0.9999781152574067, + "pass@32": 0.9999999396281514, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9973724270197736, + "pass@4": 0.9986252298245676, + "pass@8": 0.9994650709406782, + "pass@16": 0.9999226464955803, + "pass@32": 0.9999990371708989, + "pass@64": 0.9999999999948634, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5862015625, + "avg_response_len": 353.01473984375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5862015625, + "pass@2": 0.6560753444881785, + "pass@4": 0.7131012058867533, + "pass@8": 0.7581952127508048, + "pass@16": 0.7928651214680502, + "pass@32": 0.8204131462875605, + "pass@64": 0.8439453504290559, + "pass@128": 0.8649 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.965203125, + "pass@2": 0.9761099901574805, + "pass@4": 0.982154178946382, + "pass@8": 0.9855331140582771, + "pass@16": 0.9874028703702856, + "pass@32": 0.988775775461155, + "pass@64": 0.9902493279275325, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.90465625, + "pass@2": 0.9250632381889771, + "pass@4": 0.9372842222534677, + "pass@8": 0.9472372464852064, + "pass@16": 0.9562268508627267, + "pass@32": 0.9636262872888584, + "pass@64": 0.9698710905653103, + "pass@128": 0.975 + }, + "13": { + "pass@1": 0.8282265625, + "pass@2": 0.8762151820866151, + "pass@4": 0.9050727613423324, + "pass@8": 0.9251895123772045, + "pass@16": 0.9399500447368512, + "pass@32": 0.9508544029273686, + "pass@64": 0.9592453449065953, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.789828125, + "pass@2": 0.8522936761811026, + "pass@4": 0.8911661458567671, + "pass@8": 0.9156811133127087, + "pass@16": 0.9316251851029694, + "pass@32": 0.9430538782986673, + "pass@64": 0.952152972883393, + "pass@128": 0.96 + }, + "15": { + "pass@1": 0.7061328125, + "pass@2": 0.7831537893700792, + "pass@4": 0.8337866036745417, + "pass@8": 0.8686448482529571, + "pass@16": 0.893500510389893, + "pass@32": 0.9123562071153283, + "pass@64": 0.9275996177301614, + "pass@128": 0.941 + }, + "16": { + "pass@1": 0.5146875, + "pass@2": 0.6214413139763781, + "pass@4": 0.703013568522684, + "pass@8": 0.7635755842787482, + "pass@16": 0.80759708261457, + "pass@32": 0.8406243353930557, + "pass@64": 0.8690552116232023, + "pass@128": 0.895 + }, + "17": { + "pass@1": 0.400578125, + "pass@2": 0.5098169291338588, + "pass@4": 0.6036199779715038, + "pass@8": 0.6745749067178032, + "pass@16": 0.7261218464820438, + "pass@32": 0.7667979658335959, + "pass@64": 0.803095469977794, + "pass@128": 0.839 + }, + "18": { + "pass@1": 0.322390625, + "pass@2": 0.4196706446850395, + "pass@4": 0.5103244795650541, + "pass@8": 0.5872140187173576, + "pass@16": 0.6484065409865156, + "pass@32": 0.6977423040207874, + "pass@64": 0.7388153322241718, + "pass@128": 0.772 + }, + "19": { + "pass@1": 0.219296875, + "pass@2": 0.3032326525590552, + "pass@4": 0.38662797778402686, + "pass@8": 0.46050982597322976, + "pass@16": 0.5218231942039266, + "pass@32": 0.5736300390867218, + "pass@64": 0.6182279607655419, + "pass@128": 0.657 + }, + "20": { + "pass@1": 0.211015625, + "pass@2": 0.2937560285433072, + "pass@4": 0.3779621429508806, + "pass@8": 0.45379195733457633, + "pass@16": 0.5159970889305889, + "pass@32": 0.5666702674500359, + "pass@64": 0.6111411756867772, + "pass@128": 0.652 + } + }, + "per_op_accuracy": { + "11": 0.965203125, + "12": 0.90465625, + "13": 0.8282265625, + "14": 0.789828125, + "15": 0.7061328125, + "16": 0.5146875, + "17": 0.400578125, + "18": 0.322390625, + "19": 0.219296875, + "20": 0.211015625 + }, + "per_op_avg_response_len": { + "11": 298.1568515625, + "12": 308.3597265625, + "13": 327.38540625, + "14": 351.860171875, + "15": 366.3663984375, + "16": 364.117234375, + "17": 377.8932265625, + "18": 381.595828125, + "19": 370.2647578125, + "20": 384.147796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.1568515625, + "12": 308.3597265625, + "13": 327.38540625, + "14": 351.860171875, + "15": 366.3663984375, + "16": 364.117234375, + "17": 377.8932265625, + "18": 381.595828125, + "19": 370.2647578125, + "20": 384.147796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 253740, + "answer_accuracy": 0.5891066121842496, + "avg_response_len": 335.09609026745915, + "resp_tokens_sum": 144332588, + "pass_at_k": { + "pass@1": 0.5891066121842496, + "pass@2": 0.6585626630670082, + "pass@4": 0.7149186646710035, + "pass@8": 0.7593714505885888, + "pass@16": 0.7936467790396518, + "pass@32": 0.821517305723065, + "pass@64": 0.8458358575414223, + "pass@128": 0.8665676077265974 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9626195113314447, + "pass@2": 0.9762726963485086, + "pass@4": 0.9846844493234377, + "pass@8": 0.9900801453352958, + "pass@16": 0.9932379197466509, + "pass@32": 0.9954801514064505, + "pass@64": 0.997892068688777, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.901460981638418, + "pass@2": 0.9217231221362163, + "pass@4": 0.9348357231299478, + "pass@8": 0.9465797316316302, + "pass@16": 0.9578980663743366, + "pass@32": 0.9679344063172236, + "pass@64": 0.9762589021186009, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.8427948667711599, + "pass@2": 0.8905359082269886, + "pass@4": 0.9209342809421553, + "pass@8": 0.943087610276711, + "pass@16": 0.958722214299707, + "pass@32": 0.9696280816359992, + "pass@64": 0.9768711075724734, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8018886292834891, + "pass@2": 0.8628271641278481, + "pass@4": 0.9006283793965002, + "pass@8": 0.9244972063712814, + "pass@16": 0.9406100487896942, + "pass@32": 0.9526997400967827, + "pass@64": 0.9632070158028258, + "pass@128": 0.9719626168224299 + }, + "15": { + "pass@1": 0.7418262283236994, + "pass@2": 0.8192660211187478, + "pass@4": 0.8653713786138005, + "pass@8": 0.8956872473223434, + "pass@16": 0.9181508944218568, + "pass@32": 0.9362091216236437, + "pass@64": 0.9503385442837953, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.5168094758064516, + "pass@2": 0.6213535052070106, + "pass@4": 0.699477595643286, + "pass@8": 0.7560305931715449, + "pass@16": 0.7951090488831217, + "pass@32": 0.8247438496288747, + "pass@64": 0.8539612054328029, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.3988439306358382, + "pass@2": 0.5105483506667877, + "pass@4": 0.6048298715189496, + "pass@8": 0.6723869740026817, + "pass@16": 0.7184281360861899, + "pass@32": 0.7550195700251328, + "pass@64": 0.7890479343437723, + "pass@128": 0.8236994219653179 + }, + "18": { + "pass@1": 0.28816964285714286, + "pass@2": 0.3796727362204723, + "pass@4": 0.467278488403235, + "pass@8": 0.5462655241288868, + "pass@16": 0.6138568378801608, + "pass@32": 0.6703086868159625, + "pass@64": 0.7174801040252221, + "pass@128": 0.7542857142857143 + }, + "19": { + "pass@1": 0.21534816066066065, + "pass@2": 0.3016950267196331, + "pass@4": 0.3857750520486741, + "pass@8": 0.46030682718403537, + "pass@16": 0.5229785050793225, + "pass@32": 0.5764371640031789, + "pass@64": 0.6222048994664434, + "pass@128": 0.6576576576576577 + }, + "20": { + "pass@1": 0.20934215465465467, + "pass@2": 0.2943404772883118, + "pass@4": 0.38065421664634225, + "pass@8": 0.4555273052466646, + "pass@16": 0.5147219613189439, + "pass@32": 0.5640454929979681, + "pass@64": 0.608729160423394, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 254148, + "answer_accuracy": 0.5904047725245317, + "avg_response_len": 348.38111200936663, + "resp_tokens_sum": 149965527, + "pass_at_k": { + "pass@1": 0.5904047725245317, + "pass@2": 0.659745550525523, + "pass@4": 0.7167084064592308, + "pass@8": 0.7619721332534813, + "pass@16": 0.7962454943772719, + "pass@32": 0.822928673139044, + "pass@64": 0.8455948764612298, + "pass@128": 0.8661909009812667 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9662321159638554, + "pass@2": 0.9743631267194764, + "pass@4": 0.9785488667267493, + "pass@8": 0.9808695599764491, + "pass@16": 0.982073879736446, + "pass@32": 0.9826728211785501, + "pass@64": 0.9834337337433924, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9005436912225705, + "pass@2": 0.9177582658652778, + "pass@4": 0.9271106299800295, + "pass@8": 0.9343454179297891, + "pass@16": 0.940327519767136, + "pass@32": 0.9449395353999992, + "pass@64": 0.9505426950580875, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8344712885154062, + "pass@2": 0.8804364757714112, + "pass@4": 0.9060683837839594, + "pass@8": 0.9228954276841229, + "pass@16": 0.9346244711335276, + "pass@32": 0.9439304631235373, + "pass@64": 0.9527515537231487, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.7992964181286549, + "pass@2": 0.8605686213565414, + "pass@4": 0.8981318897637794, + "pass@8": 0.9218447647781702, + "pass@16": 0.9365818806667606, + "pass@32": 0.9455783721012618, + "pass@64": 0.95243629760834, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6910511363636364, + "pass@2": 0.7729125656167979, + "pass@4": 0.8302869189646748, + "pass@8": 0.869345424003105, + "pass@16": 0.8949531927840826, + "pass@32": 0.9140074088483217, + "pass@64": 0.9312807378510449, + "pass@128": 0.9484848484848485 + }, + "16": { + "pass@1": 0.5259081196581197, + "pass@2": 0.6309065633622719, + "pass@4": 0.7118756023552612, + "pass@8": 0.7720004258510963, + "pass@16": 0.8156201987024397, + "pass@32": 0.8469923462911426, + "pass@64": 0.8699568794955675, + "pass@128": 0.886039886039886 + }, + "17": { + "pass@1": 0.4095496894409938, + "pass@2": 0.5180191470631387, + "pass@4": 0.611229697219524, + "pass@8": 0.683427213949164, + "pass@16": 0.7368688044556874, + "pass@32": 0.7800162649577272, + "pass@64": 0.8180155278556003, + "pass@128": 0.8509316770186336 + }, + "18": { + "pass@1": 0.34777113970588236, + "pass@2": 0.44948543886058334, + "pass@4": 0.5446689158340496, + "pass@8": 0.6258189227539268, + "pass@16": 0.6886123060488256, + "pass@32": 0.7375933805229657, + "pass@64": 0.7786859415971895, + "pass@128": 0.8117647058823529 + }, + "19": { + "pass@1": 0.21505301339285715, + "pass@2": 0.29587866458099005, + "pass@4": 0.3803635753678111, + "pass@8": 0.4571718554243951, + "pass@16": 0.5209078149489803, + "pass@32": 0.5738744800310455, + "pass@64": 0.6185575197914691, + "pass@128": 0.6607142857142857 + }, + "20": { + "pass@1": 0.21313622754491018, + "pass@2": 0.29427770934508934, + "pass@4": 0.3753525808338328, + "pass@8": 0.4483680994520108, + "pass@16": 0.5082228833902228, + "pass@32": 0.5561960787312403, + "pass@64": 0.5972451401057024, + "pass@128": 0.6347305389221557 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 242450, + "answer_accuracy": 0.5788938340464548, + "avg_response_len": 376.2051879584352, + "resp_tokens_sum": 157560752, + "pass_at_k": { + "pass@1": 0.5788938340464548, + "pass@2": 0.6497450480574889, + "pass@4": 0.7075245664203361, + "pass@8": 0.7531035795067411, + "pass@16": 0.7885868599086929, + "pass@32": 0.816692115327273, + "pass@64": 0.8403057072507439, + "pass@128": 0.8618581907090465 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9670138888888888, + "pass@2": 0.9777687945256848, + "pass@4": 0.9831185542283404, + "pass@8": 0.9853527899769424, + "pass@16": 0.9864804972292633, + "pass@32": 0.9876949377250771, + "pass@64": 0.988868025643136, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9121272935779816, + "pass@2": 0.9358054010450525, + "pass@4": 0.9498595880790128, + "pass@8": 0.9605254836942089, + "pass@16": 0.9699280017140539, + "pass@32": 0.9771920356573748, + "pass@64": 0.9818113745926464, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.8070023148148148, + "pass@2": 0.8574642145426268, + "pass@4": 0.8883589898484909, + "pass@8": 0.9100953611595731, + "pass@16": 0.9273355622857266, + "pass@32": 0.9399995973777219, + "pass@64": 0.9490467805920115, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7687314540059347, + "pass@2": 0.8338625756442908, + "pass@4": 0.8750840646032004, + "pass@8": 0.9010284881703096, + "pass@16": 0.918036665440492, + "pass@32": 0.9313040310651919, + "pass@64": 0.9413362226368941, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6833767361111112, + "pass@2": 0.7550203685476807, + "pass@4": 0.8036216772093296, + "pass@8": 0.839052687525927, + "pass@16": 0.8656967509295822, + "pass@32": 0.8852018707209927, + "pass@64": 0.8995674009787741, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.5011292404129793, + "pass@2": 0.611721308619608, + "pass@4": 0.6970713199013836, + "pass@8": 0.7617520676160321, + "pass@16": 0.810709698278014, + "pass@32": 0.8485528863124285, + "pass@64": 0.8819244048262195, + "pass@128": 0.911504424778761 + }, + "17": { + "pass@1": 0.39368411144578314, + "pass@2": 0.5010995013518642, + "pass@4": 0.5949785539796977, + "pass@8": 0.6682694301844709, + "pass@16": 0.7237167360285948, + "pass@32": 0.7662528833991333, + "pass@64": 0.8032647130281453, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.3331905241935484, + "pass@2": 0.4321294767589536, + "pass@4": 0.5212567007753061, + "pass@8": 0.5911053275351994, + "pass@16": 0.6433176247479347, + "pass@32": 0.6850081105077166, + "pass@64": 0.7191744376525762, + "pass@128": 0.7483870967741936 + }, + "19": { + "pass@1": 0.227577416918429, + "pass@2": 0.31224464459880574, + "pass@4": 0.39384508800070667, + "pass@8": 0.4641024444058891, + "pass@16": 0.5215901093343052, + "pass@32": 0.5705578192846896, + "pass@64": 0.6138924549646005, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.21056212462462462, + "pass@2": 0.29264833238750565, + "pass@4": 0.37788746789534156, + "pass@8": 0.45749675516656396, + "pass@16": 0.525069768045186, + "pass@32": 0.5798006846410763, + "pass@64": 0.6274909563678123, + "pass@128": 0.6726726726726727 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7786768092105263, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7786768092105263, + "pass@2": 0.8164385036779771, + "pass@4": 0.8471311097675107, + "pass@8": 0.8713186648641198, + "pass@16": 0.8899031795564813, + "pass@32": 0.9046776838916891, + "pass@64": 0.9172550848448685, + "pass@128": 0.9283684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.95275, + "pass@2": 0.9646134350393707, + "pass@4": 0.9731705104986875, + "pass@8": 0.9791324008089147, + "pass@16": 0.9838400852148089, + "pass@32": 0.9877436014233196, + "pass@64": 0.9902763476517468, + "pass@128": 0.991 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999375, + "pass@2": 0.9999984005905512, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99934375, + "pass@2": 0.9996381643700788, + "pass@4": 0.9998731449193852, + "pass@8": 0.9999852822033031, + "pass@16": 0.9999998535430754, + "pass@32": 0.9999999999968788, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.994640625, + "pass@2": 0.9965073818897637, + "pass@4": 0.997699450974878, + "pass@8": 0.9982503088775315, + "pass@16": 0.998566901237073, + "pass@32": 0.9990204348783889, + "pass@64": 0.9996299212598426, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989453125, + "pass@2": 0.9992125984251969, + "pass@4": 0.9993964112298462, + "pass@8": 0.9996420801536683, + "pass@16": 0.9998812336914318, + "pass@32": 0.9999901022858957, + "pass@64": 0.999999987930507, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9975, + "pass@2": 0.9983570374015749, + "pass@4": 0.9989147164416947, + "pass@8": 0.9993051158777021, + "pass@16": 0.9996133627852174, + "pass@32": 0.9998739015543171, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951796875, + "pass@2": 0.9972514763779529, + "pass@4": 0.9985841584176979, + "pass@8": 0.9992542527706686, + "pass@16": 0.9996094514666293, + "pass@32": 0.9998738852946448, + "pass@64": 0.9999934280775866, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.994546875, + "pass@2": 0.995999630905512, + "pass@4": 0.9968406347019122, + "pass@8": 0.9975330642182132, + "pass@16": 0.9979983089545647, + "pass@32": 0.9982426056328664, + "pass@64": 0.9984999947647286, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.965203125, + "pass@2": 0.9761099901574805, + "pass@4": 0.982154178946382, + "pass@8": 0.9855331140582771, + "pass@16": 0.9874028703702856, + "pass@32": 0.988775775461155, + "pass@64": 0.9902493279275325, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.90465625, + "pass@2": 0.9250632381889771, + "pass@4": 0.9372842222534677, + "pass@8": 0.9472372464852064, + "pass@16": 0.9562268508627267, + "pass@32": 0.9636262872888584, + "pass@64": 0.9698710905653103, + "pass@128": 0.975 + }, + "13": { + "pass@1": 0.8282265625, + "pass@2": 0.8762151820866151, + "pass@4": 0.9050727613423324, + "pass@8": 0.9251895123772045, + "pass@16": 0.9399500447368512, + "pass@32": 0.9508544029273686, + "pass@64": 0.9592453449065953, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.789828125, + "pass@2": 0.8522936761811026, + "pass@4": 0.8911661458567671, + "pass@8": 0.9156811133127087, + "pass@16": 0.9316251851029694, + "pass@32": 0.9430538782986673, + "pass@64": 0.952152972883393, + "pass@128": 0.96 + }, + "15": { + "pass@1": 0.7061328125, + "pass@2": 0.7831537893700792, + "pass@4": 0.8337866036745417, + "pass@8": 0.8686448482529571, + "pass@16": 0.893500510389893, + "pass@32": 0.9123562071153283, + "pass@64": 0.9275996177301614, + "pass@128": 0.941 + }, + "16": { + "pass@1": 0.5146875, + "pass@2": 0.6214413139763781, + "pass@4": 0.703013568522684, + "pass@8": 0.7635755842787482, + "pass@16": 0.80759708261457, + "pass@32": 0.8406243353930557, + "pass@64": 0.8690552116232023, + "pass@128": 0.895 + }, + "17": { + "pass@1": 0.400578125, + "pass@2": 0.5098169291338588, + "pass@4": 0.6036199779715038, + "pass@8": 0.6745749067178032, + "pass@16": 0.7261218464820438, + "pass@32": 0.7667979658335959, + "pass@64": 0.803095469977794, + "pass@128": 0.839 + }, + "18": { + "pass@1": 0.322390625, + "pass@2": 0.4196706446850395, + "pass@4": 0.5103244795650541, + "pass@8": 0.5872140187173576, + "pass@16": 0.6484065409865156, + "pass@32": 0.6977423040207874, + "pass@64": 0.7388153322241718, + "pass@128": 0.772 + }, + "19": { + "pass@1": 0.219296875, + "pass@2": 0.3032326525590552, + "pass@4": 0.38662797778402686, + "pass@8": 0.46050982597322976, + "pass@16": 0.5218231942039266, + "pass@32": 0.5736300390867218, + "pass@64": 0.6182279607655419, + "pass@128": 0.657 + }, + "20": { + "pass@1": 0.211015625, + "pass@2": 0.2937560285433072, + "pass@4": 0.3779621429508806, + "pass@8": 0.45379195733457633, + "pass@16": 0.5159970889305889, + "pass@32": 0.5666702674500359, + "pass@64": 0.6111411756867772, + "pass@128": 0.652 + } + }, + "per_op_accuracy": { + "10": 0.95275, + "2": 1.0, + "3": 0.9999375, + "4": 0.99934375, + "5": 0.994640625, + "6": 0.9989453125, + "7": 0.9975, + "8": 0.9951796875, + "9": 0.994546875, + "11": 0.965203125, + "12": 0.90465625, + "13": 0.8282265625, + "14": 0.789828125, + "15": 0.7061328125, + "16": 0.5146875, + "17": 0.400578125, + "18": 0.322390625, + "19": 0.219296875, + "20": 0.211015625 + }, + "per_op_avg_response_len": { + "10": 280.125859375, + "2": 115.466796875, + "3": 144.4380546875, + "4": 159.5991484375, + "5": 172.673078125, + "6": 198.4890546875, + "7": 223.461984375, + "8": 242.948734375, + "9": 258.745484375, + "11": 298.1568515625, + "12": 308.3597265625, + "13": 327.38540625, + "14": 351.860171875, + "15": 366.3663984375, + "16": 364.117234375, + "17": 377.8932265625, + "18": 381.595828125, + "19": 370.2647578125, + "20": 384.147796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.125859375, + "2": 115.466796875, + "3": 144.4380546875, + "4": 159.5991484375, + "5": 172.673078125, + "6": 198.4890546875, + "7": 223.461984375, + "8": 242.948734375, + "9": 258.745484375, + "11": 298.1568515625, + "12": 308.3597265625, + "13": 327.38540625, + "14": 351.860171875, + "15": 366.3663984375, + "16": 364.117234375, + "17": 377.8932265625, + "18": 381.595828125, + "19": 370.2647578125, + "20": 384.147796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 630088, + "answer_accuracy": 0.7781477236800506, + "avg_response_len": 267.2140879406418, + "resp_tokens_sum": 216370729, + "pass_at_k": { + "pass@1": 0.7781477236800506, + "pass@2": 0.8161657784645723, + "pass@4": 0.8468454339291167, + "pass@8": 0.870913775619282, + "pass@16": 0.889445450240329, + "pass@32": 0.9045217537083076, + "pass@64": 0.9176365524098704, + "pass@128": 0.9287069238065128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.958218104719764, + "pass@2": 0.9703566679441621, + "pass@4": 0.977850502647346, + "pass@8": 0.9821515252532051, + "pass@16": 0.9859468847525427, + "pass@32": 0.990017335559609, + "pass@64": 0.9933073369353381, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981201171875, + "pass@2": 0.9988750307578741, + "pass@4": 0.9996035822670603, + "pass@8": 0.9999540068853223, + "pass@16": 0.999999542322111, + "pass@32": 0.9999999999902464, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9953125, + "pass@2": 0.997645997375328, + "pass@4": 0.999239501312336, + "pass@8": 0.999901179379142, + "pass@16": 0.9999982194545234, + "pass@32": 0.999999999792329, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9976487723372154, + "pass@4": 0.9981928522033549, + "pass@8": 0.9989283836936177, + "pass@16": 0.9996444122497958, + "pass@32": 0.9999703661254363, + "pass@64": 0.9999999638637934, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9970876479289941, + "pass@2": 0.9982957589805712, + "pass@4": 0.9992151179882099, + "pass@8": 0.9998073596919187, + "pass@16": 0.9999900064095701, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9971454326923077, + "pass@2": 0.9990662224914194, + "pass@4": 0.9998395386874719, + "pass@8": 0.9999942233437229, + "pass@16": 0.9999999948628073, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953125, + "pass@2": 0.9966337889541976, + "pass@4": 0.9978505357604947, + "pass@8": 0.998977259593415, + "pass@16": 0.9997108898434905, + "pass@32": 0.999980014289343, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9626195113314447, + "pass@2": 0.9762726963485086, + "pass@4": 0.9846844493234377, + "pass@8": 0.9900801453352958, + "pass@16": 0.9932379197466509, + "pass@32": 0.9954801514064505, + "pass@64": 0.997892068688777, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.901460981638418, + "pass@2": 0.9217231221362163, + "pass@4": 0.9348357231299478, + "pass@8": 0.9465797316316302, + "pass@16": 0.9578980663743366, + "pass@32": 0.9679344063172236, + "pass@64": 0.9762589021186009, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.8427948667711599, + "pass@2": 0.8905359082269886, + "pass@4": 0.9209342809421553, + "pass@8": 0.943087610276711, + "pass@16": 0.958722214299707, + "pass@32": 0.9696280816359992, + "pass@64": 0.9768711075724734, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8018886292834891, + "pass@2": 0.8628271641278481, + "pass@4": 0.9006283793965002, + "pass@8": 0.9244972063712814, + "pass@16": 0.9406100487896942, + "pass@32": 0.9526997400967827, + "pass@64": 0.9632070158028258, + "pass@128": 0.9719626168224299 + }, + "15": { + "pass@1": 0.7418262283236994, + "pass@2": 0.8192660211187478, + "pass@4": 0.8653713786138005, + "pass@8": 0.8956872473223434, + "pass@16": 0.9181508944218568, + "pass@32": 0.9362091216236437, + "pass@64": 0.9503385442837953, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.5168094758064516, + "pass@2": 0.6213535052070106, + "pass@4": 0.699477595643286, + "pass@8": 0.7560305931715449, + "pass@16": 0.7951090488831217, + "pass@32": 0.8247438496288747, + "pass@64": 0.8539612054328029, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.3988439306358382, + "pass@2": 0.5105483506667877, + "pass@4": 0.6048298715189496, + "pass@8": 0.6723869740026817, + "pass@16": 0.7184281360861899, + "pass@32": 0.7550195700251328, + "pass@64": 0.7890479343437723, + "pass@128": 0.8236994219653179 + }, + "18": { + "pass@1": 0.28816964285714286, + "pass@2": 0.3796727362204723, + "pass@4": 0.467278488403235, + "pass@8": 0.5462655241288868, + "pass@16": 0.6138568378801608, + "pass@32": 0.6703086868159625, + "pass@64": 0.7174801040252221, + "pass@128": 0.7542857142857143 + }, + "19": { + "pass@1": 0.21534816066066065, + "pass@2": 0.3016950267196331, + "pass@4": 0.3857750520486741, + "pass@8": 0.46030682718403537, + "pass@16": 0.5229785050793225, + "pass@32": 0.5764371640031789, + "pass@64": 0.6222048994664434, + "pass@128": 0.6576576576576577 + }, + "20": { + "pass@1": 0.20934215465465467, + "pass@2": 0.2943404772883118, + "pass@4": 0.38065421664634225, + "pass@8": 0.4555273052466646, + "pass@16": 0.5147219613189439, + "pass@32": 0.5640454929979681, + "pass@64": 0.608729160423394, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 641641, + "answer_accuracy": 0.7814217166796571, + "avg_response_len": 276.6200214341387, + "resp_tokens_sum": 227138232, + "pass_at_k": { + "pass@1": 0.7814217166796571, + "pass@2": 0.8186168229604477, + "pass@4": 0.8490477481478055, + "pass@8": 0.8732010614866647, + "pass@16": 0.8915305173438047, + "pass@32": 0.9058368632112285, + "pass@64": 0.917962162340312, + "pass@128": 0.9289166017147311 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.94140625, + "pass@2": 0.9557011134727794, + "pass@4": 0.9657509209930045, + "pass@8": 0.9726733106639606, + "pass@16": 0.9776191432638202, + "pass@32": 0.9811505532784799, + "pass@64": 0.9835180765731945, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998816287878788, + "pass@2": 0.9999962717728466, + "pass@4": 0.9999999985797231, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9970108695652173, + "pass@2": 0.99722876012781, + "pass@4": 0.9973667938246848, + "pass@8": 0.997615447525581, + "pass@16": 0.9980647799459851, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998209169054442, + "pass@2": 0.999994359587573, + "pass@4": 0.9999999959711339, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972001445086706, + "pass@2": 0.9974181164944701, + "pass@4": 0.9976984244382747, + "pass@8": 0.9981811378621208, + "pass@16": 0.9988923141492616, + "pass@32": 0.999635564735464, + "pass@64": 0.9999810060045892, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9962367957746479, + "pass@2": 0.9971404707774204, + "pass@4": 0.9977174318879153, + "pass@8": 0.9982268251618137, + "pass@16": 0.9989203963062453, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935207709580839, + "pass@2": 0.994047338394078, + "pass@4": 0.9941055330284314, + "pass@8": 0.9941991017964071, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9662321159638554, + "pass@2": 0.9743631267194764, + "pass@4": 0.9785488667267493, + "pass@8": 0.9808695599764491, + "pass@16": 0.982073879736446, + "pass@32": 0.9826728211785501, + "pass@64": 0.9834337337433924, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9005436912225705, + "pass@2": 0.9177582658652778, + "pass@4": 0.9271106299800295, + "pass@8": 0.9343454179297891, + "pass@16": 0.940327519767136, + "pass@32": 0.9449395353999992, + "pass@64": 0.9505426950580875, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8344712885154062, + "pass@2": 0.8804364757714112, + "pass@4": 0.9060683837839594, + "pass@8": 0.9228954276841229, + "pass@16": 0.9346244711335276, + "pass@32": 0.9439304631235373, + "pass@64": 0.9527515537231487, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.7992964181286549, + "pass@2": 0.8605686213565414, + "pass@4": 0.8981318897637794, + "pass@8": 0.9218447647781702, + "pass@16": 0.9365818806667606, + "pass@32": 0.9455783721012618, + "pass@64": 0.95243629760834, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6910511363636364, + "pass@2": 0.7729125656167979, + "pass@4": 0.8302869189646748, + "pass@8": 0.869345424003105, + "pass@16": 0.8949531927840826, + "pass@32": 0.9140074088483217, + "pass@64": 0.9312807378510449, + "pass@128": 0.9484848484848485 + }, + "16": { + "pass@1": 0.5259081196581197, + "pass@2": 0.6309065633622719, + "pass@4": 0.7118756023552612, + "pass@8": 0.7720004258510963, + "pass@16": 0.8156201987024397, + "pass@32": 0.8469923462911426, + "pass@64": 0.8699568794955675, + "pass@128": 0.886039886039886 + }, + "17": { + "pass@1": 0.4095496894409938, + "pass@2": 0.5180191470631387, + "pass@4": 0.611229697219524, + "pass@8": 0.683427213949164, + "pass@16": 0.7368688044556874, + "pass@32": 0.7800162649577272, + "pass@64": 0.8180155278556003, + "pass@128": 0.8509316770186336 + }, + "18": { + "pass@1": 0.34777113970588236, + "pass@2": 0.44948543886058334, + "pass@4": 0.5446689158340496, + "pass@8": 0.6258189227539268, + "pass@16": 0.6886123060488256, + "pass@32": 0.7375933805229657, + "pass@64": 0.7786859415971895, + "pass@128": 0.8117647058823529 + }, + "19": { + "pass@1": 0.21505301339285715, + "pass@2": 0.29587866458099005, + "pass@4": 0.3803635753678111, + "pass@8": 0.4571718554243951, + "pass@16": 0.5209078149489803, + "pass@32": 0.5738744800310455, + "pass@64": 0.6185575197914691, + "pass@128": 0.6607142857142857 + }, + "20": { + "pass@1": 0.21313622754491018, + "pass@2": 0.29427770934508934, + "pass@4": 0.3753525808338328, + "pass@8": 0.4483680994520108, + "pass@16": 0.5082228833902228, + "pass@32": 0.5561960787312403, + "pass@64": 0.5972451401057024, + "pass@128": 0.6347305389221557 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 622013, + "answer_accuracy": 0.7763982365393833, + "avg_response_len": 297.3608940625499, + "resp_tokens_sum": 238231275, + "pass_at_k": { + "pass@1": 0.7763982365393833, + "pass@2": 0.8144815363514256, + "pass@4": 0.8454554347626103, + "pass@8": 0.8697985745987399, + "pass@16": 0.8886979109430378, + "pass@32": 0.9036472124114285, + "pass@64": 0.9161448330555395, + "pass@128": 0.927464451190286 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9582555970149254, + "pass@2": 0.967474512281114, + "pass@4": 0.975654895097068, + "pass@8": 0.9823627894675429, + "pass@16": 0.987761942028932, + "pass@32": 0.9918586397009777, + "pass@64": 0.9937858730382253, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998629385964912, + "pass@2": 0.9999946038817517, + "pass@4": 0.9999999958886717, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9916130514705882, + "pass@2": 0.994720501389532, + "pass@4": 0.9966101887999295, + "pass@8": 0.9973650234611066, + "pass@16": 0.9977503324339216, + "pass@32": 0.9983499305180185, + "pass@64": 0.9992704955998147, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982693829113924, + "pass@2": 0.9994506410096681, + "pass@4": 0.9999251636266987, + "pass@8": 0.9999985905112019, + "pass@16": 0.9999999997567663, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.992210960960961, + "pass@2": 0.995669513016954, + "pass@4": 0.9983319340150049, + "pass@8": 0.9996562527176672, + "pass@16": 0.9999781152574067, + "pass@32": 0.9999999396281514, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9973724270197736, + "pass@4": 0.9986252298245676, + "pass@8": 0.9994650709406782, + "pass@16": 0.9999226464955803, + "pass@32": 0.9999990371708989, + "pass@64": 0.9999999999948634, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9670138888888888, + "pass@2": 0.9777687945256848, + "pass@4": 0.9831185542283404, + "pass@8": 0.9853527899769424, + "pass@16": 0.9864804972292633, + "pass@32": 0.9876949377250771, + "pass@64": 0.988868025643136, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9121272935779816, + "pass@2": 0.9358054010450525, + "pass@4": 0.9498595880790128, + "pass@8": 0.9605254836942089, + "pass@16": 0.9699280017140539, + "pass@32": 0.9771920356573748, + "pass@64": 0.9818113745926464, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.8070023148148148, + "pass@2": 0.8574642145426268, + "pass@4": 0.8883589898484909, + "pass@8": 0.9100953611595731, + "pass@16": 0.9273355622857266, + "pass@32": 0.9399995973777219, + "pass@64": 0.9490467805920115, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7687314540059347, + "pass@2": 0.8338625756442908, + "pass@4": 0.8750840646032004, + "pass@8": 0.9010284881703096, + "pass@16": 0.918036665440492, + "pass@32": 0.9313040310651919, + "pass@64": 0.9413362226368941, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6833767361111112, + "pass@2": 0.7550203685476807, + "pass@4": 0.8036216772093296, + "pass@8": 0.839052687525927, + "pass@16": 0.8656967509295822, + "pass@32": 0.8852018707209927, + "pass@64": 0.8995674009787741, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.5011292404129793, + "pass@2": 0.611721308619608, + "pass@4": 0.6970713199013836, + "pass@8": 0.7617520676160321, + "pass@16": 0.810709698278014, + "pass@32": 0.8485528863124285, + "pass@64": 0.8819244048262195, + "pass@128": 0.911504424778761 + }, + "17": { + "pass@1": 0.39368411144578314, + "pass@2": 0.5010995013518642, + "pass@4": 0.5949785539796977, + "pass@8": 0.6682694301844709, + "pass@16": 0.7237167360285948, + "pass@32": 0.7662528833991333, + "pass@64": 0.8032647130281453, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.3331905241935484, + "pass@2": 0.4321294767589536, + "pass@4": 0.5212567007753061, + "pass@8": 0.5911053275351994, + "pass@16": 0.6433176247479347, + "pass@32": 0.6850081105077166, + "pass@64": 0.7191744376525762, + "pass@128": 0.7483870967741936 + }, + "19": { + "pass@1": 0.227577416918429, + "pass@2": 0.31224464459880574, + "pass@4": 0.39384508800070667, + "pass@8": 0.4641024444058891, + "pass@16": 0.5215901093343052, + "pass@32": 0.5705578192846896, + "pass@64": 0.6138924549646005, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.21056212462462462, + "pass@2": 0.29264833238750565, + "pass@4": 0.37788746789534156, + "pass@8": 0.45749675516656396, + "pass@16": 0.525069768045186, + "pass@32": 0.5798006846410763, + "pass@64": 0.6274909563678123, + "pass@128": 0.6726726726726727 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1eb7fb6d68634cfb5c856122b4f7d2072326e906 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7363c93d60f10bff8f23280d04ea262409463c1403cc2152a0cbacf4b9df931 +size 2644861398 diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/summary.csv b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d44bb765b06215b4486b36224b82a9def5401f3e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op9-12,0.9925381944444445,0.5862015625,0.7786768092105263,0.0,0.0,0.0,199.54979947916667,353.01473984375,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/summary.json b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fff63dcf023cf37bf16fc3f18f9096bccd2c3247 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block16-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op9-12", + "id_acc": 0.9925381944444445, + "ood_acc": 0.5862015625, + "total_acc": 0.7786768092105263, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.54979947916667, + "ood_avg_resp_len": 353.01473984375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 630088, + "answer_accuracy": 0.7781477236800506, + "avg_response_len": 267.2140879406418, + "resp_tokens_sum": 216370729, + "pass_at_k": { + "pass@1": 0.7781477236800506, + "pass@2": 0.8161657784645723, + "pass@4": 0.8468454339291167, + "pass@8": 0.870913775619282, + "pass@16": 0.889445450240329, + "pass@32": 0.9045217537083076, + "pass@64": 0.9176365524098704, + "pass@128": 0.9287069238065128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.958218104719764, + "pass@2": 0.9703566679441621, + "pass@4": 0.977850502647346, + "pass@8": 0.9821515252532051, + "pass@16": 0.9859468847525427, + "pass@32": 0.990017335559609, + "pass@64": 0.9933073369353381, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981201171875, + "pass@2": 0.9988750307578741, + "pass@4": 0.9996035822670603, + "pass@8": 0.9999540068853223, + "pass@16": 0.999999542322111, + "pass@32": 0.9999999999902464, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9953125, + "pass@2": 0.997645997375328, + "pass@4": 0.999239501312336, + "pass@8": 0.999901179379142, + "pass@16": 0.9999982194545234, + "pass@32": 0.999999999792329, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9976487723372154, + "pass@4": 0.9981928522033549, + "pass@8": 0.9989283836936177, + "pass@16": 0.9996444122497958, + "pass@32": 0.9999703661254363, + "pass@64": 0.9999999638637934, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9970876479289941, + "pass@2": 0.9982957589805712, + "pass@4": 0.9992151179882099, + "pass@8": 0.9998073596919187, + "pass@16": 0.9999900064095701, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9971454326923077, + "pass@2": 0.9990662224914194, + "pass@4": 0.9998395386874719, + "pass@8": 0.9999942233437229, + "pass@16": 0.9999999948628073, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953125, + "pass@2": 0.9966337889541976, + "pass@4": 0.9978505357604947, + "pass@8": 0.998977259593415, + "pass@16": 0.9997108898434905, + "pass@32": 0.999980014289343, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9626195113314447, + "pass@2": 0.9762726963485086, + "pass@4": 0.9846844493234377, + "pass@8": 0.9900801453352958, + "pass@16": 0.9932379197466509, + "pass@32": 0.9954801514064505, + "pass@64": 0.997892068688777, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.901460981638418, + "pass@2": 0.9217231221362163, + "pass@4": 0.9348357231299478, + "pass@8": 0.9465797316316302, + "pass@16": 0.9578980663743366, + "pass@32": 0.9679344063172236, + "pass@64": 0.9762589021186009, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.8427948667711599, + "pass@2": 0.8905359082269886, + "pass@4": 0.9209342809421553, + "pass@8": 0.943087610276711, + "pass@16": 0.958722214299707, + "pass@32": 0.9696280816359992, + "pass@64": 0.9768711075724734, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8018886292834891, + "pass@2": 0.8628271641278481, + "pass@4": 0.9006283793965002, + "pass@8": 0.9244972063712814, + "pass@16": 0.9406100487896942, + "pass@32": 0.9526997400967827, + "pass@64": 0.9632070158028258, + "pass@128": 0.9719626168224299 + }, + "15": { + "pass@1": 0.7418262283236994, + "pass@2": 0.8192660211187478, + "pass@4": 0.8653713786138005, + "pass@8": 0.8956872473223434, + "pass@16": 0.9181508944218568, + "pass@32": 0.9362091216236437, + "pass@64": 0.9503385442837953, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.5168094758064516, + "pass@2": 0.6213535052070106, + "pass@4": 0.699477595643286, + "pass@8": 0.7560305931715449, + "pass@16": 0.7951090488831217, + "pass@32": 0.8247438496288747, + "pass@64": 0.8539612054328029, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.3988439306358382, + "pass@2": 0.5105483506667877, + "pass@4": 0.6048298715189496, + "pass@8": 0.6723869740026817, + "pass@16": 0.7184281360861899, + "pass@32": 0.7550195700251328, + "pass@64": 0.7890479343437723, + "pass@128": 0.8236994219653179 + }, + "18": { + "pass@1": 0.28816964285714286, + "pass@2": 0.3796727362204723, + "pass@4": 0.467278488403235, + "pass@8": 0.5462655241288868, + "pass@16": 0.6138568378801608, + "pass@32": 0.6703086868159625, + "pass@64": 0.7174801040252221, + "pass@128": 0.7542857142857143 + }, + "19": { + "pass@1": 0.21534816066066065, + "pass@2": 0.3016950267196331, + "pass@4": 0.3857750520486741, + "pass@8": 0.46030682718403537, + "pass@16": 0.5229785050793225, + "pass@32": 0.5764371640031789, + "pass@64": 0.6222048994664434, + "pass@128": 0.6576576576576577 + }, + "20": { + "pass@1": 0.20934215465465467, + "pass@2": 0.2943404772883118, + "pass@4": 0.38065421664634225, + "pass@8": 0.4555273052466646, + "pass@16": 0.5147219613189439, + "pass@32": 0.5640454929979681, + "pass@64": 0.608729160423394, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 641641, + "answer_accuracy": 0.7814217166796571, + "avg_response_len": 276.6200214341387, + "resp_tokens_sum": 227138232, + "pass_at_k": { + "pass@1": 0.7814217166796571, + "pass@2": 0.8186168229604477, + "pass@4": 0.8490477481478055, + "pass@8": 0.8732010614866647, + "pass@16": 0.8915305173438047, + "pass@32": 0.9058368632112285, + "pass@64": 0.917962162340312, + "pass@128": 0.9289166017147311 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.94140625, + "pass@2": 0.9557011134727794, + "pass@4": 0.9657509209930045, + "pass@8": 0.9726733106639606, + "pass@16": 0.9776191432638202, + "pass@32": 0.9811505532784799, + "pass@64": 0.9835180765731945, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998816287878788, + "pass@2": 0.9999962717728466, + "pass@4": 0.9999999985797231, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9970108695652173, + "pass@2": 0.99722876012781, + "pass@4": 0.9973667938246848, + "pass@8": 0.997615447525581, + "pass@16": 0.9980647799459851, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998209169054442, + "pass@2": 0.999994359587573, + "pass@4": 0.9999999959711339, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972001445086706, + "pass@2": 0.9974181164944701, + "pass@4": 0.9976984244382747, + "pass@8": 0.9981811378621208, + "pass@16": 0.9988923141492616, + "pass@32": 0.999635564735464, + "pass@64": 0.9999810060045892, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9962367957746479, + "pass@2": 0.9971404707774204, + "pass@4": 0.9977174318879153, + "pass@8": 0.9982268251618137, + "pass@16": 0.9989203963062453, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935207709580839, + "pass@2": 0.994047338394078, + "pass@4": 0.9941055330284314, + "pass@8": 0.9941991017964071, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9662321159638554, + "pass@2": 0.9743631267194764, + "pass@4": 0.9785488667267493, + "pass@8": 0.9808695599764491, + "pass@16": 0.982073879736446, + "pass@32": 0.9826728211785501, + "pass@64": 0.9834337337433924, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9005436912225705, + "pass@2": 0.9177582658652778, + "pass@4": 0.9271106299800295, + "pass@8": 0.9343454179297891, + "pass@16": 0.940327519767136, + "pass@32": 0.9449395353999992, + "pass@64": 0.9505426950580875, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8344712885154062, + "pass@2": 0.8804364757714112, + "pass@4": 0.9060683837839594, + "pass@8": 0.9228954276841229, + "pass@16": 0.9346244711335276, + "pass@32": 0.9439304631235373, + "pass@64": 0.9527515537231487, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.7992964181286549, + "pass@2": 0.8605686213565414, + "pass@4": 0.8981318897637794, + "pass@8": 0.9218447647781702, + "pass@16": 0.9365818806667606, + "pass@32": 0.9455783721012618, + "pass@64": 0.95243629760834, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6910511363636364, + "pass@2": 0.7729125656167979, + "pass@4": 0.8302869189646748, + "pass@8": 0.869345424003105, + "pass@16": 0.8949531927840826, + "pass@32": 0.9140074088483217, + "pass@64": 0.9312807378510449, + "pass@128": 0.9484848484848485 + }, + "16": { + "pass@1": 0.5259081196581197, + "pass@2": 0.6309065633622719, + "pass@4": 0.7118756023552612, + "pass@8": 0.7720004258510963, + "pass@16": 0.8156201987024397, + "pass@32": 0.8469923462911426, + "pass@64": 0.8699568794955675, + "pass@128": 0.886039886039886 + }, + "17": { + "pass@1": 0.4095496894409938, + "pass@2": 0.5180191470631387, + "pass@4": 0.611229697219524, + "pass@8": 0.683427213949164, + "pass@16": 0.7368688044556874, + "pass@32": 0.7800162649577272, + "pass@64": 0.8180155278556003, + "pass@128": 0.8509316770186336 + }, + "18": { + "pass@1": 0.34777113970588236, + "pass@2": 0.44948543886058334, + "pass@4": 0.5446689158340496, + "pass@8": 0.6258189227539268, + "pass@16": 0.6886123060488256, + "pass@32": 0.7375933805229657, + "pass@64": 0.7786859415971895, + "pass@128": 0.8117647058823529 + }, + "19": { + "pass@1": 0.21505301339285715, + "pass@2": 0.29587866458099005, + "pass@4": 0.3803635753678111, + "pass@8": 0.4571718554243951, + "pass@16": 0.5209078149489803, + "pass@32": 0.5738744800310455, + "pass@64": 0.6185575197914691, + "pass@128": 0.6607142857142857 + }, + "20": { + "pass@1": 0.21313622754491018, + "pass@2": 0.29427770934508934, + "pass@4": 0.3753525808338328, + "pass@8": 0.4483680994520108, + "pass@16": 0.5082228833902228, + "pass@32": 0.5561960787312403, + "pass@64": 0.5972451401057024, + "pass@128": 0.6347305389221557 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 622013, + "answer_accuracy": 0.7763982365393833, + "avg_response_len": 297.3608940625499, + "resp_tokens_sum": 238231275, + "pass_at_k": { + "pass@1": 0.7763982365393833, + "pass@2": 0.8144815363514256, + "pass@4": 0.8454554347626103, + "pass@8": 0.8697985745987399, + "pass@16": 0.8886979109430378, + "pass@32": 0.9036472124114285, + "pass@64": 0.9161448330555395, + "pass@128": 0.927464451190286 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9582555970149254, + "pass@2": 0.967474512281114, + "pass@4": 0.975654895097068, + "pass@8": 0.9823627894675429, + "pass@16": 0.987761942028932, + "pass@32": 0.9918586397009777, + "pass@64": 0.9937858730382253, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998629385964912, + "pass@2": 0.9999946038817517, + "pass@4": 0.9999999958886717, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9916130514705882, + "pass@2": 0.994720501389532, + "pass@4": 0.9966101887999295, + "pass@8": 0.9973650234611066, + "pass@16": 0.9977503324339216, + "pass@32": 0.9983499305180185, + "pass@64": 0.9992704955998147, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982693829113924, + "pass@2": 0.9994506410096681, + "pass@4": 0.9999251636266987, + "pass@8": 0.9999985905112019, + "pass@16": 0.9999999997567663, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.992210960960961, + "pass@2": 0.995669513016954, + "pass@4": 0.9983319340150049, + "pass@8": 0.9996562527176672, + "pass@16": 0.9999781152574067, + "pass@32": 0.9999999396281514, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9973724270197736, + "pass@4": 0.9986252298245676, + "pass@8": 0.9994650709406782, + "pass@16": 0.9999226464955803, + "pass@32": 0.9999990371708989, + "pass@64": 0.9999999999948634, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9670138888888888, + "pass@2": 0.9777687945256848, + "pass@4": 0.9831185542283404, + "pass@8": 0.9853527899769424, + "pass@16": 0.9864804972292633, + "pass@32": 0.9876949377250771, + "pass@64": 0.988868025643136, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9121272935779816, + "pass@2": 0.9358054010450525, + "pass@4": 0.9498595880790128, + "pass@8": 0.9605254836942089, + "pass@16": 0.9699280017140539, + "pass@32": 0.9771920356573748, + "pass@64": 0.9818113745926464, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.8070023148148148, + "pass@2": 0.8574642145426268, + "pass@4": 0.8883589898484909, + "pass@8": 0.9100953611595731, + "pass@16": 0.9273355622857266, + "pass@32": 0.9399995973777219, + "pass@64": 0.9490467805920115, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7687314540059347, + "pass@2": 0.8338625756442908, + "pass@4": 0.8750840646032004, + "pass@8": 0.9010284881703096, + "pass@16": 0.918036665440492, + "pass@32": 0.9313040310651919, + "pass@64": 0.9413362226368941, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6833767361111112, + "pass@2": 0.7550203685476807, + "pass@4": 0.8036216772093296, + "pass@8": 0.839052687525927, + "pass@16": 0.8656967509295822, + "pass@32": 0.8852018707209927, + "pass@64": 0.8995674009787741, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.5011292404129793, + "pass@2": 0.611721308619608, + "pass@4": 0.6970713199013836, + "pass@8": 0.7617520676160321, + "pass@16": 0.810709698278014, + "pass@32": 0.8485528863124285, + "pass@64": 0.8819244048262195, + "pass@128": 0.911504424778761 + }, + "17": { + "pass@1": 0.39368411144578314, + "pass@2": 0.5010995013518642, + "pass@4": 0.5949785539796977, + "pass@8": 0.6682694301844709, + "pass@16": 0.7237167360285948, + "pass@32": 0.7662528833991333, + "pass@64": 0.8032647130281453, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.3331905241935484, + "pass@2": 0.4321294767589536, + "pass@4": 0.5212567007753061, + "pass@8": 0.5911053275351994, + "pass@16": 0.6433176247479347, + "pass@32": 0.6850081105077166, + "pass@64": 0.7191744376525762, + "pass@128": 0.7483870967741936 + }, + "19": { + "pass@1": 0.227577416918429, + "pass@2": 0.31224464459880574, + "pass@4": 0.39384508800070667, + "pass@8": 0.4641024444058891, + "pass@16": 0.5215901093343052, + "pass@32": 0.5705578192846896, + "pass@64": 0.6138924549646005, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.21056212462462462, + "pass@2": 0.29264833238750565, + "pass@4": 0.37788746789534156, + "pass@8": 0.45749675516656396, + "pass@16": 0.525069768045186, + "pass@32": 0.5798006846410763, + "pass@64": 0.6274909563678123, + "pass@128": 0.6726726726726727 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..215a56ca49b9dcc7220c2fd31a39a01263977715 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3f17057f3af63037e425aab2296e296ac47e5f8edb8378809e780df4f70cc39 +size 1791267837 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..90235aef412ee4a36d613ec962b1e2b8f2bcb164 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op11-14", + "id": { + "answer_accuracy": 0.9832118055555555, + "avg_response_len": 199.74794444444444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9832118055555555, + "pass@2": 0.987714348206471, + "pass@4": 0.9902540394013241, + "pass@8": 0.9920937192308773, + "pass@16": 0.9936641086143041, + "pass@32": 0.9950526379297164, + "pass@64": 0.9962001383731773, + "pass@128": 0.9971111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.93071875, + "pass@2": 0.9445115649606306, + "pass@4": 0.9538971672290966, + "pass@8": 0.9613346554272514, + "pass@16": 0.9679780046457843, + "pass@32": 0.9739687630757281, + "pass@64": 0.978899009017973, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999991387795276, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99875, + "pass@2": 0.999439345472441, + "pass@4": 0.9997683861079865, + "pass@8": 0.9999505973411476, + "pass@16": 0.9999981090802724, + "pass@32": 0.999999998952485, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965390625, + "pass@2": 0.9973762303149606, + "pass@4": 0.9978447050993625, + "pass@8": 0.9983960337382891, + "pass@16": 0.999030698449425, + "pass@32": 0.9995940512152047, + "pass@64": 0.9999373449166382, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9812109375, + "pass@2": 0.9868863188976378, + "pass@4": 0.9908712279715037, + "pass@8": 0.9939349668366426, + "pass@16": 0.9960502609983615, + "pass@32": 0.99736027955402, + "pass@64": 0.9982225120759813, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9933203125, + "pass@2": 0.9944953248031496, + "pass@4": 0.9951551601049868, + "pass@8": 0.9956318795685197, + "pass@16": 0.9959852920686115, + "pass@32": 0.9962362390924381, + "pass@64": 0.996499972481556, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9879765625, + "pass@2": 0.9927413877952754, + "pass@4": 0.9950933667041619, + "pass@8": 0.9963219837356966, + "pass@16": 0.9973261114274652, + "pass@32": 0.9983528969097358, + "pass@64": 0.9993787663810586, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.982125, + "pass@2": 0.98863656496063, + "pass@4": 0.9915293319272591, + "pass@8": 0.9933862631362872, + "pass@16": 0.9951571583453197, + "pass@32": 0.9967002385265373, + "pass@64": 0.9976709488867418, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.978328125, + "pass@2": 0.9853432578740163, + "pass@4": 0.9881270095613048, + "pass@8": 0.9898870932940579, + "pass@16": 0.991451342513493, + "pass@32": 0.9932612740413003, + "pass@64": 0.995192691598646, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.93071875, + "2": 0.9999375, + "3": 0.99875, + "4": 0.9965390625, + "5": 0.9812109375, + "6": 0.9933203125, + "7": 0.9879765625, + "8": 0.982125, + "9": 0.978328125 + }, + "per_op_avg_response_len": { + "10": 279.4522734375, + "2": 115.3735546875, + "3": 144.49909375, + "4": 159.783375, + "5": 174.0365078125, + "6": 198.799171875, + "7": 223.661890625, + "8": 243.450421875, + "9": 258.6752109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.4522734375, + "2": 115.3735546875, + "3": 144.49909375, + "4": 159.783375, + "5": 174.0365078125, + "6": 198.799171875, + "7": 223.661890625, + "8": 243.450421875, + "9": 258.6752109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 372529, + "answer_accuracy": 0.9829053740290442, + "avg_response_len": 190.18669262917933, + "resp_tokens_sum": 72082278, + "pass_at_k": { + "pass@1": 0.9829053740290442, + "pass@2": 0.987678942060434, + "pass@4": 0.9902508573523048, + "pass@8": 0.9920062533264151, + "pass@16": 0.9934595256216463, + "pass@32": 0.9948872563014174, + "pass@64": 0.9961327445511798, + "pass@128": 0.9969604863221885 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9278899336283186, + "pass@2": 0.9440308311848188, + "pass@4": 0.954870106391568, + "pass@8": 0.9619631105223748, + "pass@16": 0.9677499224210235, + "pass@32": 0.9738788078996593, + "pass@64": 0.9793069762260315, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999641628440367, + "pass@2": 0.9999604944737414, + "pass@4": 0.9999996087071684, + "pass@8": 0.9999999999862356, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.9943109467273622, + "pass@4": 0.9948456068967942, + "pass@8": 0.9957570698926508, + "pass@16": 0.9971413375339783, + "pass@32": 0.9987370559545614, + "pass@64": 0.9998042033768909, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9830109126984127, + "pass@2": 0.9880233720784903, + "pass@4": 0.9914399881562425, + "pass@8": 0.9933475799961803, + "pass@16": 0.9940132950927529, + "pass@32": 0.9944443029178063, + "pass@64": 0.9952380952380618, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9924447979041916, + "pass@2": 0.9937876461643642, + "pass@4": 0.9940049877747318, + "pass@8": 0.9940119707290874, + "pass@16": 0.9940119760479038, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9898298816568047, + "pass@2": 0.9942550115314724, + "pass@4": 0.9962724605800014, + "pass@8": 0.997188033297683, + "pass@16": 0.9977253146030929, + "pass@32": 0.9983401494591545, + "pass@64": 0.9992661790057307, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9791165865384616, + "pass@2": 0.9864259192661012, + "pass@4": 0.9897139375006009, + "pass@8": 0.9928198476761539, + "pass@16": 0.9958820358520993, + "pass@32": 0.99840629029697, + "pass@64": 0.9997892603060836, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9824603873239437, + "pass@2": 0.9897149134967284, + "pass@4": 0.9922514195408672, + "pass@8": 0.9939969735757618, + "pass@16": 0.9955511666990909, + "pass@32": 0.9969822637073111, + "pass@64": 0.9984237433109547, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 384137, + "answer_accuracy": 0.9833126843053736, + "avg_response_len": 197.81795236730014, + "resp_tokens_sum": 77278770, + "pass_at_k": { + "pass@1": 0.9833126843053736, + "pass@2": 0.9868064442059428, + "pass@4": 0.9888047403928705, + "pass@8": 0.9904648736278183, + "pass@16": 0.992058759505224, + "pass@32": 0.9935460055283901, + "pass@64": 0.9949203231669694, + "pass@128": 0.9960681520314548 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9189992331288344, + "pass@2": 0.9286734336505483, + "pass@4": 0.9375544975973092, + "pass@8": 0.9470405437650068, + "pass@16": 0.9565635672147648, + "pass@32": 0.9650583970862092, + "pass@64": 0.9721751954756576, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988162878787878, + "pass@2": 0.9998038952517297, + "pass@4": 0.9999920859324403, + "pass@8": 0.9999999875941591, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9984282544378699, + "pass@2": 0.9997430170526023, + "pass@4": 0.9999895944959543, + "pass@8": 0.999999983254233, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9849411231884058, + "pass@2": 0.990598967248659, + "pass@4": 0.9934713457013521, + "pass@8": 0.9954857547548753, + "pass@16": 0.9971854860393266, + "pass@32": 0.9983304664726601, + "pass@64": 0.9992810266418577, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967317335243553, + "pass@2": 0.9971177492498251, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9916681719653179, + "pass@2": 0.9938057553138229, + "pass@4": 0.994562489569584, + "pass@8": 0.995080609648261, + "pass@16": 0.9958600718371238, + "pass@32": 0.9971688871261034, + "pass@64": 0.9989304082654408, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840228873239436, + "pass@2": 0.9901713430187427, + "pass@4": 0.9927322325026274, + "pass@8": 0.9939443822467154, + "pass@16": 0.994848759069009, + "pass@32": 0.9955955460714563, + "pass@64": 0.9964844172021773, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9740129116766467, + "pass@2": 0.9791008534112875, + "pass@4": 0.9817642866685076, + "pass@8": 0.9837510621928831, + "pass@16": 0.9854961799078583, + "pass@32": 0.9874625948756771, + "pass@64": 0.9893229611451189, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375994, + "answer_accuracy": 0.9834124958151992, + "avg_response_len": 211.1979620020087, + "resp_tokens_sum": 80748584, + "pass_at_k": { + "pass@1": 0.9834124958151992, + "pass@2": 0.9886771070307293, + "pass@4": 0.9917380309048252, + "pass@8": 0.9938447146522504, + "pass@16": 0.9955071938912271, + "pass@32": 0.9967559981875811, + "pass@64": 0.997574610792421, + "pass@128": 0.9983260796786073 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9449860074626866, + "pass@2": 0.9604106681161121, + "pass@4": 0.9688162237183038, + "pass@8": 0.9746087871426082, + "pass@16": 0.9793165911434148, + "pass@32": 0.982730775366086, + "pass@64": 0.9850293443351756, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999107142857143, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9978361880466472, + "pass@2": 0.9985917736048299, + "pass@4": 0.9993327279971927, + "pass@8": 0.9998559808733993, + "pass@16": 0.9999944871145016, + "pass@32": 0.9999999969460205, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9972816154970761, + "pass@2": 0.99790522689598, + "pass@4": 0.9985310758852513, + "pass@8": 0.9992800498032463, + "pass@16": 0.9998405568378804, + "pass@32": 0.9999947172799563, + "pass@64": 0.999999999520565, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9757582720588235, + "pass@2": 0.9820656264474293, + "pass@4": 0.987705933633296, + "pass@8": 0.992905569845335, + "pass@16": 0.9967855304722849, + "pass@32": 0.9990773917701273, + "pass@64": 0.9999134055427968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9904869873817035, + "pass@2": 0.9923538059812713, + "pass@4": 0.9941877409251287, + "pass@8": 0.9956841682808706, + "pass@16": 0.9967990286076082, + "pass@32": 0.9975906595975965, + "pass@64": 0.9984226261247826, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9819521360759493, + "pass@2": 0.9899569701235921, + "pass@4": 0.9944134608767257, + "pass@8": 0.9967548656417136, + "pass@16": 0.9985043361897948, + "pass@32": 0.9996629476168034, + "pass@64": 0.999990115877466, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9829204204204204, + "pass@2": 0.9890716257596179, + "pass@4": 0.9919478705364531, + "pass@8": 0.9933219668580872, + "pass@16": 0.994806767837737, + "pass@32": 0.996279450746294, + "pass@64": 0.9969511458392512, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.978245578778135, + "pass@2": 0.9870571657341064, + "pass@4": 0.9902523597331685, + "pass@8": 0.9917855720329248, + "pass@16": 0.9931670554536689, + "pass@32": 0.9952413624975187, + "pass@64": 0.9978083398739146, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.61535, + "avg_response_len": 366.563509375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.61535, + "pass@2": 0.6872795767716412, + "pass@4": 0.7428777255342986, + "pass@8": 0.7865097233498798, + "pass@16": 0.8209229647267159, + "pass@32": 0.848026843865256, + "pass@64": 0.8695346130196704, + "pass@128": 0.8873 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9394609375, + "pass@2": 0.9585481053149607, + "pass@4": 0.9694869194788153, + "pass@8": 0.976356841246296, + "pass@16": 0.9805945528061865, + "pass@32": 0.983707900406302, + "pass@64": 0.9867073087007363, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.8961640625, + "pass@2": 0.9236632627952758, + "pass@4": 0.9384829082302213, + "pass@8": 0.9475674294323111, + "pass@16": 0.9538842535682758, + "pass@32": 0.958897113531193, + "pass@64": 0.9637808795441927, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.8036875, + "pass@2": 0.8526423474409457, + "pass@4": 0.8811728111173602, + "pass@8": 0.9015077431016673, + "pass@16": 0.9183441530818691, + "pass@32": 0.9320875180231136, + "pass@64": 0.9441219983919232, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.779171875, + "pass@2": 0.8399190452755914, + "pass@4": 0.8757298452380947, + "pass@8": 0.8998195063240599, + "pass@16": 0.9184426109019314, + "pass@32": 0.9327292901317757, + "pass@64": 0.94250203128585, + "pass@128": 0.949 + }, + "15": { + "pass@1": 0.696296875, + "pass@2": 0.7692602116141745, + "pass@4": 0.8202982157855274, + "pass@8": 0.8584303433454398, + "pass@16": 0.886508552800874, + "pass@32": 0.9066754613115612, + "pass@64": 0.922193762834477, + "pass@128": 0.936 + }, + "16": { + "pass@1": 0.5665703125, + "pass@2": 0.6669763779527553, + "pass@4": 0.7401973246156732, + "pass@8": 0.7939764971178715, + "pass@16": 0.8328573968475546, + "pass@32": 0.8602333060153389, + "pass@64": 0.8810413764765711, + "pass@128": 0.898 + }, + "17": { + "pass@1": 0.4788515625, + "pass@2": 0.5807832185039373, + "pass@4": 0.6623702049118849, + "pass@8": 0.7252626271220843, + "pass@16": 0.7739743408468778, + "pass@32": 0.8112705613694964, + "pass@64": 0.8379079898027233, + "pass@128": 0.859 + }, + "18": { + "pass@1": 0.4408515625, + "pass@2": 0.5395221456692916, + "pass@4": 0.6196086486689172, + "pass@8": 0.6836681986446651, + "pass@16": 0.734960491800698, + "pass@32": 0.7757588523700072, + "pass@64": 0.808568882577685, + "pass@128": 0.838 + }, + "19": { + "pass@1": 0.2815859375, + "pass@2": 0.3736956200787403, + "pass@4": 0.46076443719535026, + "pass@8": 0.5377143731149852, + "pass@16": 0.6033975232991978, + "pass@32": 0.6591705088892954, + "pass@64": 0.7062790993816551, + "pass@128": 0.744 + }, + "20": { + "pass@1": 0.270859375, + "pass@2": 0.3677854330708663, + "pass@4": 0.4606659401012372, + "pass@8": 0.5407936740494892, + "pass@16": 0.606265771313584, + "pass@32": 0.6597379266044485, + "pass@64": 0.7022428012008147, + "pass@128": 0.736 + } + }, + "per_op_accuracy": { + "11": 0.9394609375, + "12": 0.8961640625, + "13": 0.8036875, + "14": 0.779171875, + "15": 0.696296875, + "16": 0.5665703125, + "17": 0.4788515625, + "18": 0.4408515625, + "19": 0.2815859375, + "20": 0.270859375 + }, + "per_op_avg_response_len": { + "11": 297.9232890625, + "12": 308.6400390625, + "13": 327.5819140625, + "14": 353.691515625, + "15": 371.24078125, + "16": 375.7909296875, + "17": 396.6282265625, + "18": 408.7279296875, + "19": 403.7931328125, + "20": 421.6173359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.9232890625, + "12": 308.6400390625, + "13": 327.5819140625, + "14": 353.691515625, + "15": 371.24078125, + "16": 375.7909296875, + "17": 396.6282265625, + "18": 408.7279296875, + "19": 403.7931328125, + "20": 421.6173359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 268445, + "answer_accuracy": 0.6232471210995543, + "avg_response_len": 348.15277906760775, + "resp_tokens_sum": 149956365, + "pass_at_k": { + "pass@1": 0.6232471210995543, + "pass@2": 0.693040877900111, + "pass@4": 0.745503823129096, + "pass@8": 0.7863127197456534, + "pass@16": 0.819345752636818, + "pass@32": 0.8463508109868225, + "pass@64": 0.8680668307545506, + "pass@128": 0.8855869242199108 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9379205028328612, + "pass@2": 0.9571442612254913, + "pass@4": 0.9700182938357914, + "pass@8": 0.9790091824786014, + "pass@16": 0.9843616470702989, + "pass@32": 0.988042761987036, + "pass@64": 0.9915216523465933, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.8998278601694916, + "pass@2": 0.9294761916677787, + "pass@4": 0.9451151222622588, + "pass@8": 0.9545353432773896, + "pass@16": 0.9606194839151998, + "pass@32": 0.964590410501115, + "pass@64": 0.9682327459739501, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8256514498432602, + "pass@2": 0.8750590088366698, + "pass@4": 0.9032777920784976, + "pass@8": 0.9225351874378085, + "pass@16": 0.9360201542065824, + "pass@32": 0.9446046406699042, + "pass@64": 0.9510021501037829, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.8089223130841121, + "pass@2": 0.8656668537052026, + "pass@4": 0.8963246548036634, + "pass@8": 0.9158750799713077, + "pass@16": 0.933101958318332, + "pass@32": 0.9476818546073893, + "pass@64": 0.9576630140348259, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7146179552023122, + "pass@2": 0.7846436780301309, + "pass@4": 0.8323375439702986, + "pass@8": 0.8690223114672836, + "pass@16": 0.897447633671605, + "pass@32": 0.9184880348028696, + "pass@64": 0.9335417669014138, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.5795110887096774, + "pass@2": 0.6735371158242319, + "pass@4": 0.7363138372824367, + "pass@8": 0.7799640125830374, + "pass@16": 0.8134978864655507, + "pass@32": 0.8406881606156303, + "pass@64": 0.8625414301962189, + "pass@128": 0.8806451612903226 + }, + "17": { + "pass@1": 0.4967711343930636, + "pass@2": 0.6005434009603571, + "pass@4": 0.681101012481821, + "pass@8": 0.7412967623110543, + "pass@16": 0.7888348559504231, + "pass@32": 0.8288105517291113, + "pass@64": 0.8597170170620166, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4057589285714286, + "pass@2": 0.5023980596175481, + "pass@4": 0.5830887969360969, + "pass@8": 0.6497490874585578, + "pass@16": 0.7036033480722533, + "pass@32": 0.7438669576288413, + "pass@64": 0.7751107452337934, + "pass@128": 0.8057142857142857 + }, + "19": { + "pass@1": 0.2859891141141141, + "pass@2": 0.3786606488378141, + "pass@4": 0.4609712226287028, + "pass@8": 0.5283011682078244, + "pass@16": 0.5859342761664774, + "pass@32": 0.6395311562863607, + "pass@64": 0.6879472864139098, + "pass@128": 0.7267267267267268 + }, + "20": { + "pass@1": 0.2658126876876877, + "pass@2": 0.3548006668085406, + "pass@4": 0.4392666600796523, + "pass@8": 0.5161147462275131, + "pass@16": 0.5838677750352207, + "pass@32": 0.6417287448192514, + "pass@64": 0.6885082490325086, + "pass@128": 0.7267267267267268 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 264828, + "answer_accuracy": 0.6152152096342551, + "avg_response_len": 362.35195045346416, + "resp_tokens_sum": 155979470, + "pass_at_k": { + "pass@1": 0.6152152096342551, + "pass@2": 0.688034709588603, + "pass@4": 0.7446000334648192, + "pass@8": 0.7894193525331746, + "pass@16": 0.8251575957322725, + "pass@32": 0.8531444454526957, + "pass@64": 0.8746369623799997, + "pass@128": 0.8917633065715135 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9419004141566265, + "pass@2": 0.960085914524239, + "pass@4": 0.969361919556742, + "pass@8": 0.974791128219019, + "pass@16": 0.9783189591002187, + "pass@32": 0.981429264922337, + "pass@64": 0.984963390203506, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8918740203761756, + "pass@2": 0.9150770277441811, + "pass@4": 0.9265427678711006, + "pass@8": 0.9332226059225804, + "pass@16": 0.9384172360559216, + "pass@32": 0.9433440083728458, + "pass@64": 0.9490673728444968, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.7813375350140056, + "pass@2": 0.8272148426299653, + "pass@4": 0.8562079947464546, + "pass@8": 0.8813857319572577, + "pass@16": 0.9051228573355422, + "pass@32": 0.9252264054744831, + "pass@64": 0.9416107030613462, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.7636147660818714, + "pass@2": 0.8289973724501539, + "pass@4": 0.8688566231852594, + "pass@8": 0.8962188338563833, + "pass@16": 0.916170300385008, + "pass@32": 0.9298274579116791, + "pass@64": 0.938172111475192, + "pass@128": 0.9444444444444444 + }, + "15": { + "pass@1": 0.6955492424242424, + "pass@2": 0.7743471874254353, + "pass@4": 0.8290293324129934, + "pass@8": 0.8680754738741443, + "pass@16": 0.8954043573712911, + "pass@32": 0.914309143504422, + "pass@64": 0.9271898072118726, + "pass@128": 0.9363636363636364 + }, + "16": { + "pass@1": 0.561698717948718, + "pass@2": 0.6652467219642416, + "pass@4": 0.7411547343975156, + "pass@8": 0.7981958322172271, + "pass@16": 0.8400369711121151, + "pass@32": 0.8682493890298797, + "pass@64": 0.8878274079235919, + "pass@128": 0.9031339031339032 + }, + "17": { + "pass@1": 0.49776785714285715, + "pass@2": 0.5991102759573528, + "pass@4": 0.6797955259474546, + "pass@8": 0.7404273567154415, + "pass@16": 0.784303732321846, + "pass@32": 0.8152163750091305, + "pass@64": 0.8362526286279424, + "pass@128": 0.8540372670807453 + }, + "18": { + "pass@1": 0.4727251838235294, + "pass@2": 0.575820692450208, + "pass@4": 0.6608747667570962, + "pass@8": 0.7297410946662416, + "pass@16": 0.7862562497922118, + "pass@32": 0.8317310767472348, + "pass@64": 0.8641656425008116, + "pass@128": 0.888235294117647 + }, + "19": { + "pass@1": 0.27487909226190477, + "pass@2": 0.36546981920228705, + "pass@4": 0.45295085240461014, + "pass@8": 0.5339778638945452, + "pass@16": 0.6060932599866877, + "pass@32": 0.6674607280697001, + "pass@64": 0.7190044214821586, + "pass@128": 0.7589285714285714 + }, + "20": { + "pass@1": 0.2742561751497006, + "pass@2": 0.3707881972275921, + "pass@4": 0.46218962609090014, + "pass@8": 0.5381715042491917, + "pass@16": 0.6003958743672162, + "pass@32": 0.6527667715200763, + "pass@64": 0.6956846611796309, + "pass@128": 0.7305389221556886 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 254375, + "answer_accuracy": 0.6073669582823961, + "avg_response_len": 389.82621724098414, + "resp_tokens_sum": 163265457, + "pass_at_k": { + "pass@1": 0.6073669582823961, + "pass@2": 0.6805783879083844, + "pass@4": 0.7384067781086533, + "pass@8": 0.7837217753623876, + "pass@16": 0.818192602443942, + "pass@32": 0.8444905836260554, + "pass@64": 0.8657998594815376, + "pass@128": 0.8844743276283619 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9386160714285714, + "pass@2": 0.9585004999375082, + "pass@4": 0.9690231887680705, + "pass@8": 0.9750347468655091, + "pass@16": 0.9787714189495187, + "pass@32": 0.9812517126052769, + "pass@64": 0.9831502218248404, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8963828363914373, + "pass@2": 0.9257465415733583, + "pass@4": 0.9429511070290525, + "pass@8": 0.9540180630667007, + "pass@16": 0.9616815228148514, + "pass@32": 0.9679063288772485, + "pass@64": 0.9733149710459309, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.806688850308642, + "pass@2": 0.8585889654418193, + "pass@4": 0.8869164855550462, + "pass@8": 0.9029762716057576, + "pass@16": 0.9155088389542627, + "pass@32": 0.9273234904167389, + "pass@64": 0.9401150972713455, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7666218471810089, + "pass@2": 0.8264774001729013, + "pass@4": 0.8630880294265885, + "pass@8": 0.8881803100130173, + "pass@16": 0.9067852805640201, + "pass@32": 0.9214315258071499, + "pass@64": 0.9324550197512018, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6774932484567902, + "pass@2": 0.7476510097696119, + "pass@4": 0.7985485983812205, + "pass@8": 0.8372954234546021, + "pass@16": 0.865766153141894, + "pass@32": 0.8862857528188541, + "pass@64": 0.9049866515637937, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.559780604719764, + "pass@2": 0.6627677658932013, + "pass@4": 0.742757295824747, + "pass@8": 0.8024215814421329, + "pass@16": 0.8431270654362341, + "pass@32": 0.8698066096607839, + "pass@64": 0.8909324865326317, + "pass@128": 0.9085545722713865 + }, + "17": { + "pass@1": 0.44182981927710846, + "pass@2": 0.5424147376909209, + "pass@4": 0.6259490820424553, + "pass@8": 0.6938443930725537, + "pass@16": 0.7484689122903563, + "pass@32": 0.7891639690309752, + "pass@64": 0.8167848056658652, + "pass@128": 0.8373493975903614 + }, + "18": { + "pass@1": 0.4455141129032258, + "pass@2": 0.5416251270002543, + "pass@4": 0.6155811259479657, + "pass@8": 0.6714324059601536, + "pass@16": 0.7141038549872825, + "pass@32": 0.7503769390511386, + "pass@64": 0.785367107405101, + "pass@128": 0.8193548387096774 + }, + "19": { + "pass@1": 0.2839643126888218, + "pass@2": 0.3770506488331708, + "pass@4": 0.4684880170756603, + "pass@8": 0.5509774073510945, + "pass@16": 0.618229830755994, + "pass@32": 0.670513079197939, + "pass@64": 0.7118041310810201, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.27249906156156156, + "pass@2": 0.37775841786668535, + "pass@4": 0.48053695849955697, + "pass@8": 0.5681026460555456, + "pass@16": 0.6345512918564712, + "pass@32": 0.6847391978736695, + "pass@64": 0.7225551875044784, + "pass@128": 0.7507507507507507 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7896003289473684, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7896003289473684, + "pass@2": 0.829590784293389, + "pass@4": 0.8600559794712976, + "pass@8": 0.8838916161356298, + "pass@16": 0.9027477170945154, + "pass@32": 0.9176706410536951, + "pass@64": 0.929534072397644, + "pass@128": 0.9393157894736842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.93071875, + "pass@2": 0.9445115649606306, + "pass@4": 0.9538971672290966, + "pass@8": 0.9613346554272514, + "pass@16": 0.9679780046457843, + "pass@32": 0.9739687630757281, + "pass@64": 0.978899009017973, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999991387795276, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99875, + "pass@2": 0.999439345472441, + "pass@4": 0.9997683861079865, + "pass@8": 0.9999505973411476, + "pass@16": 0.9999981090802724, + "pass@32": 0.999999998952485, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965390625, + "pass@2": 0.9973762303149606, + "pass@4": 0.9978447050993625, + "pass@8": 0.9983960337382891, + "pass@16": 0.999030698449425, + "pass@32": 0.9995940512152047, + "pass@64": 0.9999373449166382, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9812109375, + "pass@2": 0.9868863188976378, + "pass@4": 0.9908712279715037, + "pass@8": 0.9939349668366426, + "pass@16": 0.9960502609983615, + "pass@32": 0.99736027955402, + "pass@64": 0.9982225120759813, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9933203125, + "pass@2": 0.9944953248031496, + "pass@4": 0.9951551601049868, + "pass@8": 0.9956318795685197, + "pass@16": 0.9959852920686115, + "pass@32": 0.9962362390924381, + "pass@64": 0.996499972481556, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.9879765625, + "pass@2": 0.9927413877952754, + "pass@4": 0.9950933667041619, + "pass@8": 0.9963219837356966, + "pass@16": 0.9973261114274652, + "pass@32": 0.9983528969097358, + "pass@64": 0.9993787663810586, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.982125, + "pass@2": 0.98863656496063, + "pass@4": 0.9915293319272591, + "pass@8": 0.9933862631362872, + "pass@16": 0.9951571583453197, + "pass@32": 0.9967002385265373, + "pass@64": 0.9976709488867418, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.978328125, + "pass@2": 0.9853432578740163, + "pass@4": 0.9881270095613048, + "pass@8": 0.9898870932940579, + "pass@16": 0.991451342513493, + "pass@32": 0.9932612740413003, + "pass@64": 0.995192691598646, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9394609375, + "pass@2": 0.9585481053149607, + "pass@4": 0.9694869194788153, + "pass@8": 0.976356841246296, + "pass@16": 0.9805945528061865, + "pass@32": 0.983707900406302, + "pass@64": 0.9867073087007363, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.8961640625, + "pass@2": 0.9236632627952758, + "pass@4": 0.9384829082302213, + "pass@8": 0.9475674294323111, + "pass@16": 0.9538842535682758, + "pass@32": 0.958897113531193, + "pass@64": 0.9637808795441927, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.8036875, + "pass@2": 0.8526423474409457, + "pass@4": 0.8811728111173602, + "pass@8": 0.9015077431016673, + "pass@16": 0.9183441530818691, + "pass@32": 0.9320875180231136, + "pass@64": 0.9441219983919232, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.779171875, + "pass@2": 0.8399190452755914, + "pass@4": 0.8757298452380947, + "pass@8": 0.8998195063240599, + "pass@16": 0.9184426109019314, + "pass@32": 0.9327292901317757, + "pass@64": 0.94250203128585, + "pass@128": 0.949 + }, + "15": { + "pass@1": 0.696296875, + "pass@2": 0.7692602116141745, + "pass@4": 0.8202982157855274, + "pass@8": 0.8584303433454398, + "pass@16": 0.886508552800874, + "pass@32": 0.9066754613115612, + "pass@64": 0.922193762834477, + "pass@128": 0.936 + }, + "16": { + "pass@1": 0.5665703125, + "pass@2": 0.6669763779527553, + "pass@4": 0.7401973246156732, + "pass@8": 0.7939764971178715, + "pass@16": 0.8328573968475546, + "pass@32": 0.8602333060153389, + "pass@64": 0.8810413764765711, + "pass@128": 0.898 + }, + "17": { + "pass@1": 0.4788515625, + "pass@2": 0.5807832185039373, + "pass@4": 0.6623702049118849, + "pass@8": 0.7252626271220843, + "pass@16": 0.7739743408468778, + "pass@32": 0.8112705613694964, + "pass@64": 0.8379079898027233, + "pass@128": 0.859 + }, + "18": { + "pass@1": 0.4408515625, + "pass@2": 0.5395221456692916, + "pass@4": 0.6196086486689172, + "pass@8": 0.6836681986446651, + "pass@16": 0.734960491800698, + "pass@32": 0.7757588523700072, + "pass@64": 0.808568882577685, + "pass@128": 0.838 + }, + "19": { + "pass@1": 0.2815859375, + "pass@2": 0.3736956200787403, + "pass@4": 0.46076443719535026, + "pass@8": 0.5377143731149852, + "pass@16": 0.6033975232991978, + "pass@32": 0.6591705088892954, + "pass@64": 0.7062790993816551, + "pass@128": 0.744 + }, + "20": { + "pass@1": 0.270859375, + "pass@2": 0.3677854330708663, + "pass@4": 0.4606659401012372, + "pass@8": 0.5407936740494892, + "pass@16": 0.606265771313584, + "pass@32": 0.6597379266044485, + "pass@64": 0.7022428012008147, + "pass@128": 0.736 + } + }, + "per_op_accuracy": { + "10": 0.93071875, + "2": 0.9999375, + "3": 0.99875, + "4": 0.9965390625, + "5": 0.9812109375, + "6": 0.9933203125, + "7": 0.9879765625, + "8": 0.982125, + "9": 0.978328125, + "11": 0.9394609375, + "12": 0.8961640625, + "13": 0.8036875, + "14": 0.779171875, + "15": 0.696296875, + "16": 0.5665703125, + "17": 0.4788515625, + "18": 0.4408515625, + "19": 0.2815859375, + "20": 0.270859375 + }, + "per_op_avg_response_len": { + "10": 279.4522734375, + "2": 115.3735546875, + "3": 144.49909375, + "4": 159.783375, + "5": 174.0365078125, + "6": 198.799171875, + "7": 223.661890625, + "8": 243.450421875, + "9": 258.6752109375, + "11": 297.9232890625, + "12": 308.6400390625, + "13": 327.5819140625, + "14": 353.691515625, + "15": 371.24078125, + "16": 375.7909296875, + "17": 396.6282265625, + "18": 408.7279296875, + "19": 403.7931328125, + "20": 421.6173359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.4522734375, + "2": 115.3735546875, + "3": 144.49909375, + "4": 159.783375, + "5": 174.0365078125, + "6": 198.799171875, + "7": 223.661890625, + "8": 243.450421875, + "9": 258.6752109375, + "11": 297.9232890625, + "12": 308.6400390625, + "13": 327.5819140625, + "14": 353.691515625, + "15": 371.24078125, + "16": 375.7909296875, + "17": 396.6282265625, + "18": 408.7279296875, + "19": 403.7931328125, + "20": 421.6173359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 640974, + "answer_accuracy": 0.791591744388239, + "avg_response_len": 274.2138631738065, + "resp_tokens_sum": 222038643, + "pass_at_k": { + "pass@1": 0.791591744388239, + "pass@2": 0.8309516126422308, + "pass@4": 0.8600621488222489, + "pass@8": 0.8825913401902613, + "pass@16": 0.9008428885533657, + "pass@32": 0.9158760108882659, + "pass@64": 0.9280104239812107, + "pass@128": 0.9377173569396143 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9278899336283186, + "pass@2": 0.9440308311848188, + "pass@4": 0.954870106391568, + "pass@8": 0.9619631105223748, + "pass@16": 0.9677499224210235, + "pass@32": 0.9738788078996593, + "pass@64": 0.9793069762260315, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999641628440367, + "pass@2": 0.9999604944737414, + "pass@4": 0.9999996087071684, + "pass@8": 0.9999999999862356, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.9943109467273622, + "pass@4": 0.9948456068967942, + "pass@8": 0.9957570698926508, + "pass@16": 0.9971413375339783, + "pass@32": 0.9987370559545614, + "pass@64": 0.9998042033768909, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9830109126984127, + "pass@2": 0.9880233720784903, + "pass@4": 0.9914399881562425, + "pass@8": 0.9933475799961803, + "pass@16": 0.9940132950927529, + "pass@32": 0.9944443029178063, + "pass@64": 0.9952380952380618, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9924447979041916, + "pass@2": 0.9937876461643642, + "pass@4": 0.9940049877747318, + "pass@8": 0.9940119707290874, + "pass@16": 0.9940119760479038, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9898298816568047, + "pass@2": 0.9942550115314724, + "pass@4": 0.9962724605800014, + "pass@8": 0.997188033297683, + "pass@16": 0.9977253146030929, + "pass@32": 0.9983401494591545, + "pass@64": 0.9992661790057307, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9791165865384616, + "pass@2": 0.9864259192661012, + "pass@4": 0.9897139375006009, + "pass@8": 0.9928198476761539, + "pass@16": 0.9958820358520993, + "pass@32": 0.99840629029697, + "pass@64": 0.9997892603060836, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9824603873239437, + "pass@2": 0.9897149134967284, + "pass@4": 0.9922514195408672, + "pass@8": 0.9939969735757618, + "pass@16": 0.9955511666990909, + "pass@32": 0.9969822637073111, + "pass@64": 0.9984237433109547, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9379205028328612, + "pass@2": 0.9571442612254913, + "pass@4": 0.9700182938357914, + "pass@8": 0.9790091824786014, + "pass@16": 0.9843616470702989, + "pass@32": 0.988042761987036, + "pass@64": 0.9915216523465933, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.8998278601694916, + "pass@2": 0.9294761916677787, + "pass@4": 0.9451151222622588, + "pass@8": 0.9545353432773896, + "pass@16": 0.9606194839151998, + "pass@32": 0.964590410501115, + "pass@64": 0.9682327459739501, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8256514498432602, + "pass@2": 0.8750590088366698, + "pass@4": 0.9032777920784976, + "pass@8": 0.9225351874378085, + "pass@16": 0.9360201542065824, + "pass@32": 0.9446046406699042, + "pass@64": 0.9510021501037829, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.8089223130841121, + "pass@2": 0.8656668537052026, + "pass@4": 0.8963246548036634, + "pass@8": 0.9158750799713077, + "pass@16": 0.933101958318332, + "pass@32": 0.9476818546073893, + "pass@64": 0.9576630140348259, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7146179552023122, + "pass@2": 0.7846436780301309, + "pass@4": 0.8323375439702986, + "pass@8": 0.8690223114672836, + "pass@16": 0.897447633671605, + "pass@32": 0.9184880348028696, + "pass@64": 0.9335417669014138, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.5795110887096774, + "pass@2": 0.6735371158242319, + "pass@4": 0.7363138372824367, + "pass@8": 0.7799640125830374, + "pass@16": 0.8134978864655507, + "pass@32": 0.8406881606156303, + "pass@64": 0.8625414301962189, + "pass@128": 0.8806451612903226 + }, + "17": { + "pass@1": 0.4967711343930636, + "pass@2": 0.6005434009603571, + "pass@4": 0.681101012481821, + "pass@8": 0.7412967623110543, + "pass@16": 0.7888348559504231, + "pass@32": 0.8288105517291113, + "pass@64": 0.8597170170620166, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4057589285714286, + "pass@2": 0.5023980596175481, + "pass@4": 0.5830887969360969, + "pass@8": 0.6497490874585578, + "pass@16": 0.7036033480722533, + "pass@32": 0.7438669576288413, + "pass@64": 0.7751107452337934, + "pass@128": 0.8057142857142857 + }, + "19": { + "pass@1": 0.2859891141141141, + "pass@2": 0.3786606488378141, + "pass@4": 0.4609712226287028, + "pass@8": 0.5283011682078244, + "pass@16": 0.5859342761664774, + "pass@32": 0.6395311562863607, + "pass@64": 0.6879472864139098, + "pass@128": 0.7267267267267268 + }, + "20": { + "pass@1": 0.2658126876876877, + "pass@2": 0.3548006668085406, + "pass@4": 0.4392666600796523, + "pass@8": 0.5161147462275131, + "pass@16": 0.5838677750352207, + "pass@32": 0.6417287448192514, + "pass@64": 0.6885082490325086, + "pass@128": 0.7267267267267268 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 648965, + "answer_accuracy": 0.7903412412314887, + "avg_response_len": 284.0732657833203, + "resp_tokens_sum": 233258240, + "pass_at_k": { + "pass@1": 0.7903412412314887, + "pass@2": 0.8301783314205666, + "pass@4": 0.8607828496057989, + "pass@8": 0.8850687571131917, + "pass@16": 0.9045624830019652, + "pass@32": 0.9199418829197272, + "pass@64": 0.9318629666078854, + "pass@128": 0.9413873733437257 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9189992331288344, + "pass@2": 0.9286734336505483, + "pass@4": 0.9375544975973092, + "pass@8": 0.9470405437650068, + "pass@16": 0.9565635672147648, + "pass@32": 0.9650583970862092, + "pass@64": 0.9721751954756576, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988162878787878, + "pass@2": 0.9998038952517297, + "pass@4": 0.9999920859324403, + "pass@8": 0.9999999875941591, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9984282544378699, + "pass@2": 0.9997430170526023, + "pass@4": 0.9999895944959543, + "pass@8": 0.999999983254233, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9849411231884058, + "pass@2": 0.990598967248659, + "pass@4": 0.9934713457013521, + "pass@8": 0.9954857547548753, + "pass@16": 0.9971854860393266, + "pass@32": 0.9983304664726601, + "pass@64": 0.9992810266418577, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967317335243553, + "pass@2": 0.9971177492498251, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9916681719653179, + "pass@2": 0.9938057553138229, + "pass@4": 0.994562489569584, + "pass@8": 0.995080609648261, + "pass@16": 0.9958600718371238, + "pass@32": 0.9971688871261034, + "pass@64": 0.9989304082654408, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840228873239436, + "pass@2": 0.9901713430187427, + "pass@4": 0.9927322325026274, + "pass@8": 0.9939443822467154, + "pass@16": 0.994848759069009, + "pass@32": 0.9955955460714563, + "pass@64": 0.9964844172021773, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9740129116766467, + "pass@2": 0.9791008534112875, + "pass@4": 0.9817642866685076, + "pass@8": 0.9837510621928831, + "pass@16": 0.9854961799078583, + "pass@32": 0.9874625948756771, + "pass@64": 0.9893229611451189, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9419004141566265, + "pass@2": 0.960085914524239, + "pass@4": 0.969361919556742, + "pass@8": 0.974791128219019, + "pass@16": 0.9783189591002187, + "pass@32": 0.981429264922337, + "pass@64": 0.984963390203506, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8918740203761756, + "pass@2": 0.9150770277441811, + "pass@4": 0.9265427678711006, + "pass@8": 0.9332226059225804, + "pass@16": 0.9384172360559216, + "pass@32": 0.9433440083728458, + "pass@64": 0.9490673728444968, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.7813375350140056, + "pass@2": 0.8272148426299653, + "pass@4": 0.8562079947464546, + "pass@8": 0.8813857319572577, + "pass@16": 0.9051228573355422, + "pass@32": 0.9252264054744831, + "pass@64": 0.9416107030613462, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.7636147660818714, + "pass@2": 0.8289973724501539, + "pass@4": 0.8688566231852594, + "pass@8": 0.8962188338563833, + "pass@16": 0.916170300385008, + "pass@32": 0.9298274579116791, + "pass@64": 0.938172111475192, + "pass@128": 0.9444444444444444 + }, + "15": { + "pass@1": 0.6955492424242424, + "pass@2": 0.7743471874254353, + "pass@4": 0.8290293324129934, + "pass@8": 0.8680754738741443, + "pass@16": 0.8954043573712911, + "pass@32": 0.914309143504422, + "pass@64": 0.9271898072118726, + "pass@128": 0.9363636363636364 + }, + "16": { + "pass@1": 0.561698717948718, + "pass@2": 0.6652467219642416, + "pass@4": 0.7411547343975156, + "pass@8": 0.7981958322172271, + "pass@16": 0.8400369711121151, + "pass@32": 0.8682493890298797, + "pass@64": 0.8878274079235919, + "pass@128": 0.9031339031339032 + }, + "17": { + "pass@1": 0.49776785714285715, + "pass@2": 0.5991102759573528, + "pass@4": 0.6797955259474546, + "pass@8": 0.7404273567154415, + "pass@16": 0.784303732321846, + "pass@32": 0.8152163750091305, + "pass@64": 0.8362526286279424, + "pass@128": 0.8540372670807453 + }, + "18": { + "pass@1": 0.4727251838235294, + "pass@2": 0.575820692450208, + "pass@4": 0.6608747667570962, + "pass@8": 0.7297410946662416, + "pass@16": 0.7862562497922118, + "pass@32": 0.8317310767472348, + "pass@64": 0.8641656425008116, + "pass@128": 0.888235294117647 + }, + "19": { + "pass@1": 0.27487909226190477, + "pass@2": 0.36546981920228705, + "pass@4": 0.45295085240461014, + "pass@8": 0.5339778638945452, + "pass@16": 0.6060932599866877, + "pass@32": 0.6674607280697001, + "pass@64": 0.7190044214821586, + "pass@128": 0.7589285714285714 + }, + "20": { + "pass@1": 0.2742561751497006, + "pass@2": 0.3707881972275921, + "pass@4": 0.46218962609090014, + "pass@8": 0.5381715042491917, + "pass@16": 0.6003958743672162, + "pass@32": 0.6527667715200763, + "pass@64": 0.6956846611796309, + "pass@128": 0.7305389221556886 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630369, + "answer_accuracy": 0.7868282173669915, + "avg_response_len": 304.5789575511264, + "resp_tokens_sum": 244014041, + "pass_at_k": { + "pass@1": 0.7868282173669915, + "pass@2": 0.8276131976253347, + "pass@4": 0.8593047573548769, + "pass@8": 0.8839993308279217, + "pass@16": 0.9028129387042159, + "pass@32": 0.9171566314444449, + "pass@64": 0.9286870910146343, + "pass@128": 0.9388081163125099 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9449860074626866, + "pass@2": 0.9604106681161121, + "pass@4": 0.9688162237183038, + "pass@8": 0.9746087871426082, + "pass@16": 0.9793165911434148, + "pass@32": 0.982730775366086, + "pass@64": 0.9850293443351756, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999107142857143, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9978361880466472, + "pass@2": 0.9985917736048299, + "pass@4": 0.9993327279971927, + "pass@8": 0.9998559808733993, + "pass@16": 0.9999944871145016, + "pass@32": 0.9999999969460205, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9972816154970761, + "pass@2": 0.99790522689598, + "pass@4": 0.9985310758852513, + "pass@8": 0.9992800498032463, + "pass@16": 0.9998405568378804, + "pass@32": 0.9999947172799563, + "pass@64": 0.999999999520565, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9757582720588235, + "pass@2": 0.9820656264474293, + "pass@4": 0.987705933633296, + "pass@8": 0.992905569845335, + "pass@16": 0.9967855304722849, + "pass@32": 0.9990773917701273, + "pass@64": 0.9999134055427968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9904869873817035, + "pass@2": 0.9923538059812713, + "pass@4": 0.9941877409251287, + "pass@8": 0.9956841682808706, + "pass@16": 0.9967990286076082, + "pass@32": 0.9975906595975965, + "pass@64": 0.9984226261247826, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9819521360759493, + "pass@2": 0.9899569701235921, + "pass@4": 0.9944134608767257, + "pass@8": 0.9967548656417136, + "pass@16": 0.9985043361897948, + "pass@32": 0.9996629476168034, + "pass@64": 0.999990115877466, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9829204204204204, + "pass@2": 0.9890716257596179, + "pass@4": 0.9919478705364531, + "pass@8": 0.9933219668580872, + "pass@16": 0.994806767837737, + "pass@32": 0.996279450746294, + "pass@64": 0.9969511458392512, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.978245578778135, + "pass@2": 0.9870571657341064, + "pass@4": 0.9902523597331685, + "pass@8": 0.9917855720329248, + "pass@16": 0.9931670554536689, + "pass@32": 0.9952413624975187, + "pass@64": 0.9978083398739146, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9386160714285714, + "pass@2": 0.9585004999375082, + "pass@4": 0.9690231887680705, + "pass@8": 0.9750347468655091, + "pass@16": 0.9787714189495187, + "pass@32": 0.9812517126052769, + "pass@64": 0.9831502218248404, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8963828363914373, + "pass@2": 0.9257465415733583, + "pass@4": 0.9429511070290525, + "pass@8": 0.9540180630667007, + "pass@16": 0.9616815228148514, + "pass@32": 0.9679063288772485, + "pass@64": 0.9733149710459309, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.806688850308642, + "pass@2": 0.8585889654418193, + "pass@4": 0.8869164855550462, + "pass@8": 0.9029762716057576, + "pass@16": 0.9155088389542627, + "pass@32": 0.9273234904167389, + "pass@64": 0.9401150972713455, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7666218471810089, + "pass@2": 0.8264774001729013, + "pass@4": 0.8630880294265885, + "pass@8": 0.8881803100130173, + "pass@16": 0.9067852805640201, + "pass@32": 0.9214315258071499, + "pass@64": 0.9324550197512018, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6774932484567902, + "pass@2": 0.7476510097696119, + "pass@4": 0.7985485983812205, + "pass@8": 0.8372954234546021, + "pass@16": 0.865766153141894, + "pass@32": 0.8862857528188541, + "pass@64": 0.9049866515637937, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.559780604719764, + "pass@2": 0.6627677658932013, + "pass@4": 0.742757295824747, + "pass@8": 0.8024215814421329, + "pass@16": 0.8431270654362341, + "pass@32": 0.8698066096607839, + "pass@64": 0.8909324865326317, + "pass@128": 0.9085545722713865 + }, + "17": { + "pass@1": 0.44182981927710846, + "pass@2": 0.5424147376909209, + "pass@4": 0.6259490820424553, + "pass@8": 0.6938443930725537, + "pass@16": 0.7484689122903563, + "pass@32": 0.7891639690309752, + "pass@64": 0.8167848056658652, + "pass@128": 0.8373493975903614 + }, + "18": { + "pass@1": 0.4455141129032258, + "pass@2": 0.5416251270002543, + "pass@4": 0.6155811259479657, + "pass@8": 0.6714324059601536, + "pass@16": 0.7141038549872825, + "pass@32": 0.7503769390511386, + "pass@64": 0.785367107405101, + "pass@128": 0.8193548387096774 + }, + "19": { + "pass@1": 0.2839643126888218, + "pass@2": 0.3770506488331708, + "pass@4": 0.4684880170756603, + "pass@8": 0.5509774073510945, + "pass@16": 0.618229830755994, + "pass@32": 0.670513079197939, + "pass@64": 0.7118041310810201, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.27249906156156156, + "pass@2": 0.37775841786668535, + "pass@4": 0.48053695849955697, + "pass@8": 0.5681026460555456, + "pass@16": 0.6345512918564712, + "pass@32": 0.6847391978736695, + "pass@64": 0.7225551875044784, + "pass@128": 0.7507507507507507 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d63f404d596e05809015c440ad8be36f9a846dad --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17a39617ab70cd5b26f85fe5ced2c254f9cd02107e9b5e51d64c00cbfaf2cae2 +size 2686447450 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/summary.csv b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d1a5eaf23071804b5af820a0e1272db62d5fbf3c --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op11-14,0.9832118055555555,0.61535,0.7896003289473684,0.0,0.0,0.0,199.74794444444444,366.563509375,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/summary.json b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3f8f3bdb60e2fe6ce8f84c8e0c63dbea93356d21 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op11-14", + "id_acc": 0.9832118055555555, + "ood_acc": 0.61535, + "total_acc": 0.7896003289473684, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.74794444444444, + "ood_avg_resp_len": 366.563509375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 640974, + "answer_accuracy": 0.791591744388239, + "avg_response_len": 274.2138631738065, + "resp_tokens_sum": 222038643, + "pass_at_k": { + "pass@1": 0.791591744388239, + "pass@2": 0.8309516126422308, + "pass@4": 0.8600621488222489, + "pass@8": 0.8825913401902613, + "pass@16": 0.9008428885533657, + "pass@32": 0.9158760108882659, + "pass@64": 0.9280104239812107, + "pass@128": 0.9377173569396143 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9278899336283186, + "pass@2": 0.9440308311848188, + "pass@4": 0.954870106391568, + "pass@8": 0.9619631105223748, + "pass@16": 0.9677499224210235, + "pass@32": 0.9738788078996593, + "pass@64": 0.9793069762260315, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999641628440367, + "pass@2": 0.9999604944737414, + "pass@4": 0.9999996087071684, + "pass@8": 0.9999999999862356, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.9943109467273622, + "pass@4": 0.9948456068967942, + "pass@8": 0.9957570698926508, + "pass@16": 0.9971413375339783, + "pass@32": 0.9987370559545614, + "pass@64": 0.9998042033768909, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9830109126984127, + "pass@2": 0.9880233720784903, + "pass@4": 0.9914399881562425, + "pass@8": 0.9933475799961803, + "pass@16": 0.9940132950927529, + "pass@32": 0.9944443029178063, + "pass@64": 0.9952380952380618, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9924447979041916, + "pass@2": 0.9937876461643642, + "pass@4": 0.9940049877747318, + "pass@8": 0.9940119707290874, + "pass@16": 0.9940119760479038, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9898298816568047, + "pass@2": 0.9942550115314724, + "pass@4": 0.9962724605800014, + "pass@8": 0.997188033297683, + "pass@16": 0.9977253146030929, + "pass@32": 0.9983401494591545, + "pass@64": 0.9992661790057307, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9791165865384616, + "pass@2": 0.9864259192661012, + "pass@4": 0.9897139375006009, + "pass@8": 0.9928198476761539, + "pass@16": 0.9958820358520993, + "pass@32": 0.99840629029697, + "pass@64": 0.9997892603060836, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9824603873239437, + "pass@2": 0.9897149134967284, + "pass@4": 0.9922514195408672, + "pass@8": 0.9939969735757618, + "pass@16": 0.9955511666990909, + "pass@32": 0.9969822637073111, + "pass@64": 0.9984237433109547, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9379205028328612, + "pass@2": 0.9571442612254913, + "pass@4": 0.9700182938357914, + "pass@8": 0.9790091824786014, + "pass@16": 0.9843616470702989, + "pass@32": 0.988042761987036, + "pass@64": 0.9915216523465933, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.8998278601694916, + "pass@2": 0.9294761916677787, + "pass@4": 0.9451151222622588, + "pass@8": 0.9545353432773896, + "pass@16": 0.9606194839151998, + "pass@32": 0.964590410501115, + "pass@64": 0.9682327459739501, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8256514498432602, + "pass@2": 0.8750590088366698, + "pass@4": 0.9032777920784976, + "pass@8": 0.9225351874378085, + "pass@16": 0.9360201542065824, + "pass@32": 0.9446046406699042, + "pass@64": 0.9510021501037829, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.8089223130841121, + "pass@2": 0.8656668537052026, + "pass@4": 0.8963246548036634, + "pass@8": 0.9158750799713077, + "pass@16": 0.933101958318332, + "pass@32": 0.9476818546073893, + "pass@64": 0.9576630140348259, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7146179552023122, + "pass@2": 0.7846436780301309, + "pass@4": 0.8323375439702986, + "pass@8": 0.8690223114672836, + "pass@16": 0.897447633671605, + "pass@32": 0.9184880348028696, + "pass@64": 0.9335417669014138, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.5795110887096774, + "pass@2": 0.6735371158242319, + "pass@4": 0.7363138372824367, + "pass@8": 0.7799640125830374, + "pass@16": 0.8134978864655507, + "pass@32": 0.8406881606156303, + "pass@64": 0.8625414301962189, + "pass@128": 0.8806451612903226 + }, + "17": { + "pass@1": 0.4967711343930636, + "pass@2": 0.6005434009603571, + "pass@4": 0.681101012481821, + "pass@8": 0.7412967623110543, + "pass@16": 0.7888348559504231, + "pass@32": 0.8288105517291113, + "pass@64": 0.8597170170620166, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4057589285714286, + "pass@2": 0.5023980596175481, + "pass@4": 0.5830887969360969, + "pass@8": 0.6497490874585578, + "pass@16": 0.7036033480722533, + "pass@32": 0.7438669576288413, + "pass@64": 0.7751107452337934, + "pass@128": 0.8057142857142857 + }, + "19": { + "pass@1": 0.2859891141141141, + "pass@2": 0.3786606488378141, + "pass@4": 0.4609712226287028, + "pass@8": 0.5283011682078244, + "pass@16": 0.5859342761664774, + "pass@32": 0.6395311562863607, + "pass@64": 0.6879472864139098, + "pass@128": 0.7267267267267268 + }, + "20": { + "pass@1": 0.2658126876876877, + "pass@2": 0.3548006668085406, + "pass@4": 0.4392666600796523, + "pass@8": 0.5161147462275131, + "pass@16": 0.5838677750352207, + "pass@32": 0.6417287448192514, + "pass@64": 0.6885082490325086, + "pass@128": 0.7267267267267268 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 648965, + "answer_accuracy": 0.7903412412314887, + "avg_response_len": 284.0732657833203, + "resp_tokens_sum": 233258240, + "pass_at_k": { + "pass@1": 0.7903412412314887, + "pass@2": 0.8301783314205666, + "pass@4": 0.8607828496057989, + "pass@8": 0.8850687571131917, + "pass@16": 0.9045624830019652, + "pass@32": 0.9199418829197272, + "pass@64": 0.9318629666078854, + "pass@128": 0.9413873733437257 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9189992331288344, + "pass@2": 0.9286734336505483, + "pass@4": 0.9375544975973092, + "pass@8": 0.9470405437650068, + "pass@16": 0.9565635672147648, + "pass@32": 0.9650583970862092, + "pass@64": 0.9721751954756576, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988162878787878, + "pass@2": 0.9998038952517297, + "pass@4": 0.9999920859324403, + "pass@8": 0.9999999875941591, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9984282544378699, + "pass@2": 0.9997430170526023, + "pass@4": 0.9999895944959543, + "pass@8": 0.999999983254233, + "pass@16": 0.9999999999999906, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9849411231884058, + "pass@2": 0.990598967248659, + "pass@4": 0.9934713457013521, + "pass@8": 0.9954857547548753, + "pass@16": 0.9971854860393266, + "pass@32": 0.9983304664726601, + "pass@64": 0.9992810266418577, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967317335243553, + "pass@2": 0.9971177492498251, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9916681719653179, + "pass@2": 0.9938057553138229, + "pass@4": 0.994562489569584, + "pass@8": 0.995080609648261, + "pass@16": 0.9958600718371238, + "pass@32": 0.9971688871261034, + "pass@64": 0.9989304082654408, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840228873239436, + "pass@2": 0.9901713430187427, + "pass@4": 0.9927322325026274, + "pass@8": 0.9939443822467154, + "pass@16": 0.994848759069009, + "pass@32": 0.9955955460714563, + "pass@64": 0.9964844172021773, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9740129116766467, + "pass@2": 0.9791008534112875, + "pass@4": 0.9817642866685076, + "pass@8": 0.9837510621928831, + "pass@16": 0.9854961799078583, + "pass@32": 0.9874625948756771, + "pass@64": 0.9893229611451189, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9419004141566265, + "pass@2": 0.960085914524239, + "pass@4": 0.969361919556742, + "pass@8": 0.974791128219019, + "pass@16": 0.9783189591002187, + "pass@32": 0.981429264922337, + "pass@64": 0.984963390203506, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8918740203761756, + "pass@2": 0.9150770277441811, + "pass@4": 0.9265427678711006, + "pass@8": 0.9332226059225804, + "pass@16": 0.9384172360559216, + "pass@32": 0.9433440083728458, + "pass@64": 0.9490673728444968, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.7813375350140056, + "pass@2": 0.8272148426299653, + "pass@4": 0.8562079947464546, + "pass@8": 0.8813857319572577, + "pass@16": 0.9051228573355422, + "pass@32": 0.9252264054744831, + "pass@64": 0.9416107030613462, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.7636147660818714, + "pass@2": 0.8289973724501539, + "pass@4": 0.8688566231852594, + "pass@8": 0.8962188338563833, + "pass@16": 0.916170300385008, + "pass@32": 0.9298274579116791, + "pass@64": 0.938172111475192, + "pass@128": 0.9444444444444444 + }, + "15": { + "pass@1": 0.6955492424242424, + "pass@2": 0.7743471874254353, + "pass@4": 0.8290293324129934, + "pass@8": 0.8680754738741443, + "pass@16": 0.8954043573712911, + "pass@32": 0.914309143504422, + "pass@64": 0.9271898072118726, + "pass@128": 0.9363636363636364 + }, + "16": { + "pass@1": 0.561698717948718, + "pass@2": 0.6652467219642416, + "pass@4": 0.7411547343975156, + "pass@8": 0.7981958322172271, + "pass@16": 0.8400369711121151, + "pass@32": 0.8682493890298797, + "pass@64": 0.8878274079235919, + "pass@128": 0.9031339031339032 + }, + "17": { + "pass@1": 0.49776785714285715, + "pass@2": 0.5991102759573528, + "pass@4": 0.6797955259474546, + "pass@8": 0.7404273567154415, + "pass@16": 0.784303732321846, + "pass@32": 0.8152163750091305, + "pass@64": 0.8362526286279424, + "pass@128": 0.8540372670807453 + }, + "18": { + "pass@1": 0.4727251838235294, + "pass@2": 0.575820692450208, + "pass@4": 0.6608747667570962, + "pass@8": 0.7297410946662416, + "pass@16": 0.7862562497922118, + "pass@32": 0.8317310767472348, + "pass@64": 0.8641656425008116, + "pass@128": 0.888235294117647 + }, + "19": { + "pass@1": 0.27487909226190477, + "pass@2": 0.36546981920228705, + "pass@4": 0.45295085240461014, + "pass@8": 0.5339778638945452, + "pass@16": 0.6060932599866877, + "pass@32": 0.6674607280697001, + "pass@64": 0.7190044214821586, + "pass@128": 0.7589285714285714 + }, + "20": { + "pass@1": 0.2742561751497006, + "pass@2": 0.3707881972275921, + "pass@4": 0.46218962609090014, + "pass@8": 0.5381715042491917, + "pass@16": 0.6003958743672162, + "pass@32": 0.6527667715200763, + "pass@64": 0.6956846611796309, + "pass@128": 0.7305389221556886 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630369, + "answer_accuracy": 0.7868282173669915, + "avg_response_len": 304.5789575511264, + "resp_tokens_sum": 244014041, + "pass_at_k": { + "pass@1": 0.7868282173669915, + "pass@2": 0.8276131976253347, + "pass@4": 0.8593047573548769, + "pass@8": 0.8839993308279217, + "pass@16": 0.9028129387042159, + "pass@32": 0.9171566314444449, + "pass@64": 0.9286870910146343, + "pass@128": 0.9388081163125099 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9449860074626866, + "pass@2": 0.9604106681161121, + "pass@4": 0.9688162237183038, + "pass@8": 0.9746087871426082, + "pass@16": 0.9793165911434148, + "pass@32": 0.982730775366086, + "pass@64": 0.9850293443351756, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999107142857143, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9978361880466472, + "pass@2": 0.9985917736048299, + "pass@4": 0.9993327279971927, + "pass@8": 0.9998559808733993, + "pass@16": 0.9999944871145016, + "pass@32": 0.9999999969460205, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9972816154970761, + "pass@2": 0.99790522689598, + "pass@4": 0.9985310758852513, + "pass@8": 0.9992800498032463, + "pass@16": 0.9998405568378804, + "pass@32": 0.9999947172799563, + "pass@64": 0.999999999520565, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9757582720588235, + "pass@2": 0.9820656264474293, + "pass@4": 0.987705933633296, + "pass@8": 0.992905569845335, + "pass@16": 0.9967855304722849, + "pass@32": 0.9990773917701273, + "pass@64": 0.9999134055427968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9904869873817035, + "pass@2": 0.9923538059812713, + "pass@4": 0.9941877409251287, + "pass@8": 0.9956841682808706, + "pass@16": 0.9967990286076082, + "pass@32": 0.9975906595975965, + "pass@64": 0.9984226261247826, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9819521360759493, + "pass@2": 0.9899569701235921, + "pass@4": 0.9944134608767257, + "pass@8": 0.9967548656417136, + "pass@16": 0.9985043361897948, + "pass@32": 0.9996629476168034, + "pass@64": 0.999990115877466, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9829204204204204, + "pass@2": 0.9890716257596179, + "pass@4": 0.9919478705364531, + "pass@8": 0.9933219668580872, + "pass@16": 0.994806767837737, + "pass@32": 0.996279450746294, + "pass@64": 0.9969511458392512, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.978245578778135, + "pass@2": 0.9870571657341064, + "pass@4": 0.9902523597331685, + "pass@8": 0.9917855720329248, + "pass@16": 0.9931670554536689, + "pass@32": 0.9952413624975187, + "pass@64": 0.9978083398739146, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9386160714285714, + "pass@2": 0.9585004999375082, + "pass@4": 0.9690231887680705, + "pass@8": 0.9750347468655091, + "pass@16": 0.9787714189495187, + "pass@32": 0.9812517126052769, + "pass@64": 0.9831502218248404, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8963828363914373, + "pass@2": 0.9257465415733583, + "pass@4": 0.9429511070290525, + "pass@8": 0.9540180630667007, + "pass@16": 0.9616815228148514, + "pass@32": 0.9679063288772485, + "pass@64": 0.9733149710459309, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.806688850308642, + "pass@2": 0.8585889654418193, + "pass@4": 0.8869164855550462, + "pass@8": 0.9029762716057576, + "pass@16": 0.9155088389542627, + "pass@32": 0.9273234904167389, + "pass@64": 0.9401150972713455, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7666218471810089, + "pass@2": 0.8264774001729013, + "pass@4": 0.8630880294265885, + "pass@8": 0.8881803100130173, + "pass@16": 0.9067852805640201, + "pass@32": 0.9214315258071499, + "pass@64": 0.9324550197512018, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6774932484567902, + "pass@2": 0.7476510097696119, + "pass@4": 0.7985485983812205, + "pass@8": 0.8372954234546021, + "pass@16": 0.865766153141894, + "pass@32": 0.8862857528188541, + "pass@64": 0.9049866515637937, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.559780604719764, + "pass@2": 0.6627677658932013, + "pass@4": 0.742757295824747, + "pass@8": 0.8024215814421329, + "pass@16": 0.8431270654362341, + "pass@32": 0.8698066096607839, + "pass@64": 0.8909324865326317, + "pass@128": 0.9085545722713865 + }, + "17": { + "pass@1": 0.44182981927710846, + "pass@2": 0.5424147376909209, + "pass@4": 0.6259490820424553, + "pass@8": 0.6938443930725537, + "pass@16": 0.7484689122903563, + "pass@32": 0.7891639690309752, + "pass@64": 0.8167848056658652, + "pass@128": 0.8373493975903614 + }, + "18": { + "pass@1": 0.4455141129032258, + "pass@2": 0.5416251270002543, + "pass@4": 0.6155811259479657, + "pass@8": 0.6714324059601536, + "pass@16": 0.7141038549872825, + "pass@32": 0.7503769390511386, + "pass@64": 0.785367107405101, + "pass@128": 0.8193548387096774 + }, + "19": { + "pass@1": 0.2839643126888218, + "pass@2": 0.3770506488331708, + "pass@4": 0.4684880170756603, + "pass@8": 0.5509774073510945, + "pass@16": 0.618229830755994, + "pass@32": 0.670513079197939, + "pass@64": 0.7118041310810201, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.27249906156156156, + "pass@2": 0.37775841786668535, + "pass@4": 0.48053695849955697, + "pass@8": 0.5681026460555456, + "pass@16": 0.6345512918564712, + "pass@32": 0.6847391978736695, + "pass@64": 0.7225551875044784, + "pass@128": 0.7507507507507507 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ece2d50dc1bb986d0a14eb5f825ecbf9e0312d1b --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad23e8c1cfbe6cd4d2ea0f4e152a801fd19e1e3b164a63dbacd33546b3f98f22 +size 1792297930 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a57181f60ebc18733d742051b2fbb337e03f363a --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9809236111111111, + "avg_response_len": 199.76715711805556, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9809236111111111, + "pass@2": 0.9877218941382275, + "pass@4": 0.9914349205724272, + "pass@8": 0.9938707221377978, + "pass@16": 0.9956347409259318, + "pass@32": 0.9969967760785576, + "pass@64": 0.9980562539395982, + "pass@128": 0.9987777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9278125, + "pass@2": 0.9501754429133851, + "pass@4": 0.9646802129733786, + "pass@8": 0.974813414312994, + "pass@16": 0.982007756783658, + "pass@32": 0.9873755686845729, + "pass@64": 0.9915933294451101, + "pass@128": 0.995 + }, + "2": { + "pass@1": 0.9999296875, + "pass@2": 0.9999988927165354, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973203125, + "pass@2": 0.9985895669291337, + "pass@4": 0.9991034382264715, + "pass@8": 0.9993244970067723, + "pass@16": 0.9995588892955813, + "pass@32": 0.9998290717623712, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9977109375, + "pass@2": 0.9984077263779527, + "pass@4": 0.9988661218597675, + "pass@8": 0.9992694656335577, + "pass@16": 0.9996082444233209, + "pass@32": 0.9998738754052786, + "pass@64": 0.9999934280775842, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9774140625, + "pass@2": 0.9842214566929132, + "pass@4": 0.9883268593925758, + "pass@8": 0.9912612107869159, + "pass@16": 0.9935807327378486, + "pass@32": 0.9956923112441951, + "pass@64": 0.9973976556273569, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9926015625, + "pass@2": 0.994494340551181, + "pass@4": 0.9953843020247469, + "pass@8": 0.9960045319194786, + "pass@16": 0.9965902743709288, + "pass@32": 0.9972633400995086, + "pass@64": 0.9981299160261685, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9862421875, + "pass@2": 0.9928046259842522, + "pass@4": 0.995571023715785, + "pass@8": 0.9966462872730303, + "pass@16": 0.9973871136425051, + "pass@32": 0.9982022786272272, + "pass@64": 0.9991284765229548, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9763828125, + "pass@2": 0.9861296751968507, + "pass@4": 0.991461055774278, + "pass@8": 0.9950380092213381, + "pass@16": 0.9974562877679477, + "pass@32": 0.9989783810545692, + "pass@64": 0.9998214531091393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9728984375, + "pass@2": 0.9846753198818904, + "pass@4": 0.9895212712785901, + "pass@8": 0.9924790830861014, + "pass@16": 0.994523369311588, + "pass@32": 0.9957561578292949, + "pass@64": 0.9964558503469343, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9278125, + "2": 0.9999296875, + "3": 0.9973203125, + "4": 0.9977109375, + "5": 0.9774140625, + "6": 0.9926015625, + "7": 0.9862421875, + "8": 0.9763828125, + "9": 0.9728984375 + }, + "per_op_avg_response_len": { + "10": 279.173984375, + "2": 115.3352109375, + "3": 144.679890625, + "4": 159.49659375, + "5": 174.3488828125, + "6": 198.7245703125, + "7": 223.7893046875, + "8": 243.8599140625, + "9": 258.4960625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.173984375, + "2": 115.3352109375, + "3": 144.679890625, + "4": 159.49659375, + "5": 174.3488828125, + "6": 198.7245703125, + "7": 223.7893046875, + "8": 243.8599140625, + "9": 258.4960625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371462, + "answer_accuracy": 0.9800901300236406, + "avg_response_len": 190.23784194528875, + "resp_tokens_sum": 72101664, + "pass_at_k": { + "pass@1": 0.9800901300236406, + "pass@2": 0.9869469856826417, + "pass@4": 0.9907623635444761, + "pass@8": 0.9932746890034296, + "pass@16": 0.995061867371655, + "pass@32": 0.996517678060696, + "pass@64": 0.9976512893414459, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9243869837758112, + "pass@2": 0.9460098599400741, + "pass@4": 0.9614304732926082, + "pass@8": 0.9726916923371863, + "pass@16": 0.9808099201760766, + "pass@32": 0.987371006759809, + "pass@64": 0.9923337148073603, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999026479750779, + "pass@2": 0.999998850172934, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985187308868502, + "pass@2": 0.9994356353391605, + "pass@4": 0.9999022283911758, + "pass@8": 0.9999973956511242, + "pass@16": 0.9999999990253551, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959228515625, + "pass@2": 0.9968446265994094, + "pass@4": 0.997471655593832, + "pass@8": 0.9980329679238036, + "pass@16": 0.9988023146599382, + "pass@32": 0.9996059543702204, + "pass@64": 0.9999794627424621, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9770337301587302, + "pass@2": 0.9836676665416823, + "pass@4": 0.9878900095821355, + "pass@8": 0.9906436267752851, + "pass@16": 0.9923964763989225, + "pass@32": 0.9940517762081893, + "pass@64": 0.9958489186730718, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9939740351973219, + "pass@4": 0.9941051227129544, + "pass@8": 0.9941991017897541, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9886048446745562, + "pass@2": 0.9934385628756462, + "pass@4": 0.9956759880791526, + "pass@8": 0.9967927655521467, + "pass@16": 0.9973605187841855, + "pass@32": 0.9977806459943003, + "pass@64": 0.998520710058505, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9702774439102564, + "pass@2": 0.982531104885928, + "pass@4": 0.9888667660532817, + "pass@8": 0.9931890969396343, + "pass@16": 0.9965075842527735, + "pass@32": 0.9988519859805879, + "pass@64": 0.9999112970726224, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9757482394366197, + "pass@2": 0.9878105245647109, + "pass@4": 0.9923439170455804, + "pass@8": 0.9945576813789561, + "pass@16": 0.9958275331654328, + "pass@32": 0.9966991001974037, + "pass@64": 0.9971441585191202, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383307, + "answer_accuracy": 0.9811880529161205, + "avg_response_len": 197.75076025966578, + "resp_tokens_sum": 77252521, + "pass_at_k": { + "pass@1": 0.9811880529161205, + "pass@2": 0.9868467559674314, + "pass@4": 0.9901620292016258, + "pass@8": 0.9926608356932436, + "pass@16": 0.9945722980163245, + "pass@32": 0.9960414251215739, + "pass@64": 0.9973530729035318, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9198619631901841, + "pass@2": 0.9403108545480898, + "pass@4": 0.9548925379726305, + "pass@8": 0.966678628809453, + "pass@16": 0.9754899667090449, + "pass@32": 0.9819108547202062, + "pass@64": 0.9875499594029227, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974431818181818, + "pass@2": 0.9992409329515629, + "pass@4": 0.9998840809671518, + "pass@8": 0.9999964960477193, + "pass@16": 0.9999999981202536, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996995192307693, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9800271739130435, + "pass@2": 0.9857996690631067, + "pass@4": 0.9883860204974377, + "pass@8": 0.9901598054192082, + "pass@16": 0.9920850067141929, + "pass@32": 0.9943094397909279, + "pass@64": 0.996374879969186, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966048242447487, + "pass@4": 0.9971893615375443, + "pass@8": 0.9974785218231972, + "pass@16": 0.997808696404298, + "pass@32": 0.9983924824583172, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9900198699421965, + "pass@2": 0.9935380017978244, + "pass@4": 0.9945450686077535, + "pass@8": 0.995080130097074, + "pass@16": 0.9958600713338742, + "pass@32": 0.9971688871261034, + "pass@64": 0.9989304082654408, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9748679577464788, + "pass@2": 0.9831252772540756, + "pass@4": 0.9888548461371907, + "pass@8": 0.9932193457316243, + "pass@16": 0.9961788407990759, + "pass@32": 0.9981455597006466, + "pass@64": 0.9995750159591402, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9718843562874252, + "pass@2": 0.9816797261775662, + "pass@4": 0.9866750974990404, + "pass@8": 0.9905660602213172, + "pass@16": 0.9931415853062885, + "pass@32": 0.9939450571469223, + "pass@64": 0.9940117895736039, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375255, + "answer_accuracy": 0.981479640944091, + "avg_response_len": 211.27380105456982, + "resp_tokens_sum": 80777580, + "pass_at_k": { + "pass@1": 0.981479640944091, + "pass@2": 0.9893842395129568, + "pass@4": 0.9934022141189504, + "pass@8": 0.9956977819100278, + "pass@16": 0.997288190659554, + "pass@32": 0.9984478441574336, + "pass@64": 0.9991761765031066, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390158582089553, + "pass@2": 0.9639903337642501, + "pass@4": 0.9774934780540493, + "pass@8": 0.9848767093098696, + "pass@16": 0.9895625811845377, + "pass@32": 0.9926980858334635, + "pass@64": 0.9947788458509305, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9999776785714286, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996059584548105, + "pass@2": 0.9971562865866257, + "pass@4": 0.9975908537125278, + "pass@8": 0.9980364575309247, + "pass@16": 0.9987139656985613, + "pass@32": 0.9995016669456886, + "pass@64": 0.999959697670942, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974186769005848, + "pass@2": 0.9983095760233918, + "pass@4": 0.9990503037449264, + "pass@8": 0.9997044324501367, + "pass@16": 0.9999751571115222, + "pass@32": 0.9999999123006084, + "pass@64": 0.9999999999999893, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9751148897058823, + "pass@2": 0.9831331056044463, + "pass@4": 0.988671556772315, + "pass@8": 0.9929509867149238, + "pass@16": 0.996195633399387, + "pass@32": 0.998615426502192, + "pass@64": 0.9998703313410889, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.989870859621451, + "pass@2": 0.9927190193497106, + "pass@4": 0.994744807135701, + "pass@8": 0.9962839993861353, + "pass@16": 0.9975711019742235, + "pass@32": 0.9986572988061697, + "pass@64": 0.9996149764108252, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9795787183544303, + "pass@2": 0.9913235541463173, + "pass@4": 0.9965821076162948, + "pass@8": 0.9982044541228395, + "pass@16": 0.9990875747212965, + "pass@32": 0.9997847637200062, + "pass@64": 0.9999954280485364, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9837180930930931, + "pass@2": 0.9927041953370693, + "pass@4": 0.996670133294149, + "pass@8": 0.9987091388632008, + "pass@16": 0.9997070059982301, + "pass@32": 0.9999846478525416, + "pass@64": 0.9999999932642234, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9707345257234726, + "pass@2": 0.9843136820518011, + "pass@4": 0.9893559426454328, + "pass@8": 0.9921609070181736, + "pass@16": 0.9945186673490651, + "pass@32": 0.9966248494281174, + "pass@64": 0.9982949722027745, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.62520859375, + "avg_response_len": 372.23534921875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.62520859375, + "pass@2": 0.705117937992116, + "pass@4": 0.7654068212410865, + "pass@8": 0.8105699759232412, + "pass@16": 0.8444485903722265, + "pass@32": 0.8704636212704061, + "pass@64": 0.8914403410670373, + "pass@128": 0.9091 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9165703125, + "pass@2": 0.9458005659448827, + "pass@4": 0.9609296941319835, + "pass@8": 0.9697698741513148, + "pass@16": 0.975665021364242, + "pass@32": 0.9799316052674407, + "pass@64": 0.9837443745153316, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.85575, + "pass@2": 0.9035137795275596, + "pass@4": 0.9309128615485561, + "pass@8": 0.9473482917495422, + "pass@16": 0.9578326300178321, + "pass@32": 0.9650510962099749, + "pass@64": 0.9711598624469415, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.761546875, + "pass@2": 0.8256793799212615, + "pass@4": 0.8667133899512565, + "pass@8": 0.893906272562099, + "pass@16": 0.9125510312039155, + "pass@32": 0.9278046447012922, + "pass@64": 0.9418154896943188, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.7515234375, + "pass@2": 0.8211961122047252, + "pass@4": 0.8657761706974131, + "pass@8": 0.8953914309916877, + "pass@16": 0.917066297718051, + "pass@32": 0.9347341199780241, + "pass@64": 0.9482186055439786, + "pass@128": 0.958 + }, + "15": { + "pass@1": 0.69196875, + "pass@2": 0.770805118110237, + "pass@4": 0.8272369524746912, + "pass@8": 0.8681615625173151, + "pass@16": 0.8969751097309325, + "pass@32": 0.9173472774122473, + "pass@64": 0.932671814346041, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.5714609375, + "pass@2": 0.6711456692913389, + "pass@4": 0.7447089698162729, + "pass@8": 0.7980077112580461, + "pass@16": 0.8375072654978535, + "pass@32": 0.868260493343919, + "pass@64": 0.8938043019290013, + "pass@128": 0.917 + }, + "17": { + "pass@1": 0.520609375, + "pass@2": 0.6218607283464572, + "pass@4": 0.6999979126359205, + "pass@8": 0.7572542370083248, + "pass@16": 0.7993103455150429, + "pass@32": 0.832089004748486, + "pass@64": 0.858850744912394, + "pass@128": 0.881 + }, + "18": { + "pass@1": 0.4875703125, + "pass@2": 0.5872860482283465, + "pass@4": 0.6678126829771277, + "pass@8": 0.7311182672040392, + "pass@16": 0.7789627696835705, + "pass@32": 0.8137898202363205, + "pass@64": 0.8409868940100222, + "pass@128": 0.864 + }, + "19": { + "pass@1": 0.3426875, + "pass@2": 0.4421115895669299, + "pass@4": 0.5321194268841393, + "pass@8": 0.6091817086298573, + "pass@16": 0.6739818752366215, + "pass@32": 0.7277194786061778, + "pass@64": 0.7719735343932309, + "pass@128": 0.81 + }, + "20": { + "pass@1": 0.3523984375, + "pass@2": 0.46178038877952754, + "pass@4": 0.5578601512935879, + "pass@8": 0.6355604031602676, + "pass@16": 0.6946335577540641, + "pass@32": 0.7379086722001347, + "pass@64": 0.7711777888790495, + "pass@128": 0.798 + } + }, + "per_op_accuracy": { + "11": 0.9165703125, + "12": 0.85575, + "13": 0.761546875, + "14": 0.7515234375, + "15": 0.69196875, + "16": 0.5714609375, + "17": 0.520609375, + "18": 0.4875703125, + "19": 0.3426875, + "20": 0.3523984375 + }, + "per_op_avg_response_len": { + "11": 297.61453125, + "12": 307.7755703125, + "13": 325.891640625, + "14": 352.984046875, + "15": 372.9274609375, + "16": 377.635015625, + "17": 403.17771875, + "18": 418.6960234375, + "19": 419.20465625, + "20": 446.446828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.61453125, + "12": 307.7755703125, + "13": 325.891640625, + "14": 352.984046875, + "15": 372.9274609375, + "16": 377.635015625, + "17": 403.17771875, + "18": 418.6960234375, + "19": 419.20465625, + "20": 446.446828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 270011, + "answer_accuracy": 0.6268828937592867, + "avg_response_len": 354.2328705423477, + "resp_tokens_sum": 152575182, + "pass_at_k": { + "pass@1": 0.6268828937592867, + "pass@2": 0.7056139289934635, + "pass@4": 0.7645849825142607, + "pass@8": 0.8091790353541848, + "pass@16": 0.8430017687704661, + "pass@32": 0.8688598155521728, + "pass@64": 0.8901791918391903, + "pass@128": 0.9093610698365527 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9068918201133145, + "pass@2": 0.9340708020120008, + "pass@4": 0.9504890493504177, + "pass@8": 0.9617702037856002, + "pass@16": 0.9693008978007052, + "pass@32": 0.9742877622847005, + "pass@64": 0.9792593320688112, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.842867231638418, + "pass@2": 0.8940118416077222, + "pass@4": 0.9220305488826605, + "pass@8": 0.9400609534932638, + "pass@16": 0.9528386819412975, + "pass@32": 0.9611094486108223, + "pass@64": 0.9665597294441702, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.767682210031348, + "pass@2": 0.8322517617801694, + "pass@4": 0.8766375023067248, + "pass@8": 0.9080308723082444, + "pass@16": 0.9280418580337233, + "pass@32": 0.9415882651053201, + "pass@64": 0.9528324893138677, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.7715391355140186, + "pass@2": 0.8373182506684327, + "pass@4": 0.8764985632637043, + "pass@8": 0.9029473056185894, + "pass@16": 0.9240925109656627, + "pass@32": 0.9420440326754451, + "pass@64": 0.9570938658215324, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7029669436416185, + "pass@2": 0.7771679145236904, + "pass@4": 0.8295301861653124, + "pass@8": 0.868905281809576, + "pass@16": 0.897089399919766, + "pass@32": 0.9178947537440465, + "pass@64": 0.9354956627501524, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5763356854838709, + "pass@2": 0.6727306642113283, + "pass@4": 0.7455648006700768, + "pass@8": 0.798273290372829, + "pass@16": 0.8345066509257882, + "pass@32": 0.8602376954283026, + "pass@64": 0.8810953202102305, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5152637283236994, + "pass@2": 0.6144669393746302, + "pass@4": 0.6907101791105587, + "pass@8": 0.7469530410448263, + "pass@16": 0.7886757629108314, + "pass@32": 0.8232300018880613, + "pass@64": 0.8524341111586986, + "pass@128": 0.8757225433526011 + }, + "18": { + "pass@1": 0.4566071428571429, + "pass@2": 0.5588877952755906, + "pass@4": 0.6421446362954628, + "pass@8": 0.7082410808697225, + "pass@16": 0.7607307427371466, + "pass@32": 0.799389937452332, + "pass@64": 0.8314085165537201, + "pass@128": 0.8628571428571429 + }, + "19": { + "pass@1": 0.3484421921921922, + "pass@2": 0.44732589380719306, + "pass@4": 0.5323035952150127, + "pass@8": 0.6028325568129795, + "pass@16": 0.6648757853907019, + "pass@32": 0.7194209733459501, + "pass@64": 0.7657646008944604, + "pass@128": 0.8078078078078078 + }, + "20": { + "pass@1": 0.36751595345345345, + "pass@2": 0.4783597573951905, + "pass@4": 0.5728757984643807, + "pass@8": 0.6483896146476361, + "pass@16": 0.7054732379264461, + "pass@32": 0.7457587090598242, + "pass@64": 0.7768040635022134, + "pass@128": 0.8018018018018018 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 269135, + "answer_accuracy": 0.6252206920903954, + "avg_response_len": 367.6837877267321, + "resp_tokens_sum": 158274634, + "pass_at_k": { + "pass@1": 0.6252206920903954, + "pass@2": 0.7063046123750625, + "pass@4": 0.7674937128454409, + "pass@8": 0.8131793522780488, + "pass@16": 0.8475831139568042, + "pass@32": 0.874098572518286, + "pass@64": 0.894856350078689, + "pass@128": 0.9110912875408861 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9253812123493976, + "pass@2": 0.9544638939616732, + "pass@4": 0.9677863153965692, + "pass@8": 0.9736431669648126, + "pass@16": 0.9771981706043198, + "pass@32": 0.9804179898916575, + "pass@64": 0.9839934853283643, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8534972570532915, + "pass@2": 0.8976026985165256, + "pass@4": 0.9223919453250156, + "pass@8": 0.9368541385246163, + "pass@16": 0.9456598452786678, + "pass@32": 0.9524532517270197, + "pass@64": 0.9595871670233102, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7447478991596639, + "pass@2": 0.809078552680915, + "pass@4": 0.8497325329081762, + "pass@8": 0.8773586600126486, + "pass@16": 0.8976721335804148, + "pass@32": 0.9161543175079596, + "pass@64": 0.9345612772225022, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7345349049707602, + "pass@2": 0.8065829764700457, + "pass@4": 0.8545422951517018, + "pass@8": 0.8879746860407295, + "pass@16": 0.9123875941505499, + "pass@32": 0.9320443484017564, + "pass@64": 0.9469134412037941, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6902698863636364, + "pass@2": 0.7734207229778092, + "pass@4": 0.8322661962709201, + "pass@8": 0.8754337937292219, + "pass@16": 0.907189536730187, + "pass@32": 0.9293226717684716, + "pass@64": 0.9437996441836513, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5670405982905983, + "pass@2": 0.6678566721627744, + "pass@4": 0.7426079993205973, + "pass@8": 0.7965303456407977, + "pass@16": 0.8387066560797861, + "pass@32": 0.873853793076624, + "pass@64": 0.9024630188422406, + "pass@128": 0.9259259259259259 + }, + "17": { + "pass@1": 0.5341372282608695, + "pass@2": 0.6374235065046218, + "pass@4": 0.7168359166672484, + "pass@8": 0.7718156135361299, + "pass@16": 0.8099417862425061, + "pass@32": 0.8392414057749845, + "pass@64": 0.8639867822902568, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5121783088235294, + "pass@2": 0.617220067160723, + "pass@4": 0.7035296877412376, + "pass@8": 0.7725314289848646, + "pass@16": 0.8238449099525559, + "pass@32": 0.8584219170158941, + "pass@64": 0.8806343237101412, + "pass@128": 0.8941176470588236 + }, + "19": { + "pass@1": 0.34688895089285715, + "pass@2": 0.44558822237064094, + "pass@4": 0.5352641821000053, + "pass@8": 0.6142787717357525, + "pass@16": 0.6813574607373921, + "pass@32": 0.7350333718827479, + "pass@64": 0.7773131190777017, + "pass@128": 0.8125 + }, + "20": { + "pass@1": 0.34889595808383234, + "pass@2": 0.45785434544297254, + "pass@4": 0.5531497398902979, + "pass@8": 0.6277050654841096, + "pass@16": 0.6832874410114024, + "pass@32": 0.7244500613151685, + "pass@64": 0.7548202541371628, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 261121, + "answer_accuracy": 0.6234742703239609, + "avg_response_len": 395.4276603568154, + "resp_tokens_sum": 165611431, + "pass_at_k": { + "pass@1": 0.6234742703239609, + "pass@2": 0.7033881715895332, + "pass@4": 0.7641070873934189, + "pass@8": 0.8093185035313626, + "pass@16": 0.8427148348327711, + "pass@32": 0.8683769663789562, + "pass@64": 0.8892263217655131, + "pass@128": 0.9067848410757946 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9181299603174603, + "pass@2": 0.9498144763154606, + "pass@4": 0.9654031841257935, + "pass@8": 0.9746522564529527, + "pass@16": 0.9811809898411397, + "pass@32": 0.9857436588473356, + "pass@64": 0.9885079148127753, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8718941131498471, + "pass@2": 0.9195667485371669, + "pass@4": 0.948841029619004, + "pass@8": 0.9654747523656698, + "pass@16": 0.9751139020388929, + "pass@32": 0.9816078412869249, + "pass@64": 0.9874293943219232, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7740162037037037, + "pass@2": 0.8375000759453681, + "pass@4": 0.8756528162567636, + "pass@8": 0.8982326625655992, + "pass@16": 0.9136936012436747, + "pass@32": 0.927070669081338, + "pass@64": 0.9389615729467963, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7496986275964391, + "pass@2": 0.8206693643543079, + "pass@4": 0.8669634034061768, + "pass@8": 0.8957210779293493, + "pass@16": 0.9151217937643485, + "pass@32": 0.9305009446166331, + "pass@64": 0.9410892514052502, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6819540895061729, + "pass@2": 0.7613462379702535, + "pass@4": 0.8196656274447172, + "pass@8": 0.8599604416066629, + "pass@16": 0.8864494760423818, + "pass@32": 0.9045654658432458, + "pass@64": 0.9183223223885263, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.5715800147492626, + "pass@2": 0.6731016857129583, + "pass@4": 0.7461016927640676, + "pass@8": 0.7992945130458669, + "pass@16": 0.8390093434420521, + "pass@32": 0.869805682570058, + "pass@64": 0.8964608644548767, + "pass@128": 0.9233038348082596 + }, + "17": { + "pass@1": 0.5130600527108434, + "pass@2": 0.6144722838203205, + "pass@4": 0.6933464623397975, + "pass@8": 0.7538670399041609, + "pass@16": 0.8000821577644895, + "pass@32": 0.8343846127580472, + "pass@64": 0.8605566221506656, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.49553931451612904, + "pass@2": 0.5865177324104649, + "pass@4": 0.6576192465861119, + "pass@8": 0.7115264614347813, + "pass@16": 0.7503217430377445, + "pass@32": 0.7810964204600007, + "pass@64": 0.8083169134024887, + "pass@128": 0.832258064516129 + }, + "19": { + "pass@1": 0.33263311933534745, + "pass@2": 0.43333662904108294, + "pass@4": 0.5287418866825937, + "pass@8": 0.6103951658849588, + "pass@16": 0.6756559875944252, + "pass@32": 0.7286437508440294, + "pass@64": 0.7727997410431042, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3407939189189189, + "pass@2": 0.4491388534203494, + "pass@4": 0.5475690609056749, + "pass@8": 0.6306101189516876, + "pass@16": 0.6951740667469974, + "pass@32": 0.7435576624743155, + "pass@64": 0.7819581707237205, + "pass@128": 0.8168168168168168 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7937051809210526, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7937051809210526, + "pass@2": 0.8389829698507878, + "pass@4": 0.8724727630296011, + "pass@8": 0.8973966451827817, + "pass@16": 0.9160630827397604, + "pass@32": 0.9304003788111155, + "pass@64": 0.9419426155856174, + "pass@128": 0.9515789473684211 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9278125, + "pass@2": 0.9501754429133851, + "pass@4": 0.9646802129733786, + "pass@8": 0.974813414312994, + "pass@16": 0.982007756783658, + "pass@32": 0.9873755686845729, + "pass@64": 0.9915933294451101, + "pass@128": 0.995 + }, + "2": { + "pass@1": 0.9999296875, + "pass@2": 0.9999988927165354, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973203125, + "pass@2": 0.9985895669291337, + "pass@4": 0.9991034382264715, + "pass@8": 0.9993244970067723, + "pass@16": 0.9995588892955813, + "pass@32": 0.9998290717623712, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9977109375, + "pass@2": 0.9984077263779527, + "pass@4": 0.9988661218597675, + "pass@8": 0.9992694656335577, + "pass@16": 0.9996082444233209, + "pass@32": 0.9998738754052786, + "pass@64": 0.9999934280775842, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9774140625, + "pass@2": 0.9842214566929132, + "pass@4": 0.9883268593925758, + "pass@8": 0.9912612107869159, + "pass@16": 0.9935807327378486, + "pass@32": 0.9956923112441951, + "pass@64": 0.9973976556273569, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9926015625, + "pass@2": 0.994494340551181, + "pass@4": 0.9953843020247469, + "pass@8": 0.9960045319194786, + "pass@16": 0.9965902743709288, + "pass@32": 0.9972633400995086, + "pass@64": 0.9981299160261685, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9862421875, + "pass@2": 0.9928046259842522, + "pass@4": 0.995571023715785, + "pass@8": 0.9966462872730303, + "pass@16": 0.9973871136425051, + "pass@32": 0.9982022786272272, + "pass@64": 0.9991284765229548, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9763828125, + "pass@2": 0.9861296751968507, + "pass@4": 0.991461055774278, + "pass@8": 0.9950380092213381, + "pass@16": 0.9974562877679477, + "pass@32": 0.9989783810545692, + "pass@64": 0.9998214531091393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9728984375, + "pass@2": 0.9846753198818904, + "pass@4": 0.9895212712785901, + "pass@8": 0.9924790830861014, + "pass@16": 0.994523369311588, + "pass@32": 0.9957561578292949, + "pass@64": 0.9964558503469343, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9165703125, + "pass@2": 0.9458005659448827, + "pass@4": 0.9609296941319835, + "pass@8": 0.9697698741513148, + "pass@16": 0.975665021364242, + "pass@32": 0.9799316052674407, + "pass@64": 0.9837443745153316, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.85575, + "pass@2": 0.9035137795275596, + "pass@4": 0.9309128615485561, + "pass@8": 0.9473482917495422, + "pass@16": 0.9578326300178321, + "pass@32": 0.9650510962099749, + "pass@64": 0.9711598624469415, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.761546875, + "pass@2": 0.8256793799212615, + "pass@4": 0.8667133899512565, + "pass@8": 0.893906272562099, + "pass@16": 0.9125510312039155, + "pass@32": 0.9278046447012922, + "pass@64": 0.9418154896943188, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.7515234375, + "pass@2": 0.8211961122047252, + "pass@4": 0.8657761706974131, + "pass@8": 0.8953914309916877, + "pass@16": 0.917066297718051, + "pass@32": 0.9347341199780241, + "pass@64": 0.9482186055439786, + "pass@128": 0.958 + }, + "15": { + "pass@1": 0.69196875, + "pass@2": 0.770805118110237, + "pass@4": 0.8272369524746912, + "pass@8": 0.8681615625173151, + "pass@16": 0.8969751097309325, + "pass@32": 0.9173472774122473, + "pass@64": 0.932671814346041, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.5714609375, + "pass@2": 0.6711456692913389, + "pass@4": 0.7447089698162729, + "pass@8": 0.7980077112580461, + "pass@16": 0.8375072654978535, + "pass@32": 0.868260493343919, + "pass@64": 0.8938043019290013, + "pass@128": 0.917 + }, + "17": { + "pass@1": 0.520609375, + "pass@2": 0.6218607283464572, + "pass@4": 0.6999979126359205, + "pass@8": 0.7572542370083248, + "pass@16": 0.7993103455150429, + "pass@32": 0.832089004748486, + "pass@64": 0.858850744912394, + "pass@128": 0.881 + }, + "18": { + "pass@1": 0.4875703125, + "pass@2": 0.5872860482283465, + "pass@4": 0.6678126829771277, + "pass@8": 0.7311182672040392, + "pass@16": 0.7789627696835705, + "pass@32": 0.8137898202363205, + "pass@64": 0.8409868940100222, + "pass@128": 0.864 + }, + "19": { + "pass@1": 0.3426875, + "pass@2": 0.4421115895669299, + "pass@4": 0.5321194268841393, + "pass@8": 0.6091817086298573, + "pass@16": 0.6739818752366215, + "pass@32": 0.7277194786061778, + "pass@64": 0.7719735343932309, + "pass@128": 0.81 + }, + "20": { + "pass@1": 0.3523984375, + "pass@2": 0.46178038877952754, + "pass@4": 0.5578601512935879, + "pass@8": 0.6355604031602676, + "pass@16": 0.6946335577540641, + "pass@32": 0.7379086722001347, + "pass@64": 0.7711777888790495, + "pass@128": 0.798 + } + }, + "per_op_accuracy": { + "10": 0.9278125, + "2": 0.9999296875, + "3": 0.9973203125, + "4": 0.9977109375, + "5": 0.9774140625, + "6": 0.9926015625, + "7": 0.9862421875, + "8": 0.9763828125, + "9": 0.9728984375, + "11": 0.9165703125, + "12": 0.85575, + "13": 0.761546875, + "14": 0.7515234375, + "15": 0.69196875, + "16": 0.5714609375, + "17": 0.520609375, + "18": 0.4875703125, + "19": 0.3426875, + "20": 0.3523984375 + }, + "per_op_avg_response_len": { + "10": 279.173984375, + "2": 115.3352109375, + "3": 144.679890625, + "4": 159.49659375, + "5": 174.3488828125, + "6": 198.7245703125, + "7": 223.7893046875, + "8": 243.8599140625, + "9": 258.4960625, + "11": 297.61453125, + "12": 307.7755703125, + "13": 325.891640625, + "14": 352.984046875, + "15": 372.9274609375, + "16": 377.635015625, + "17": 403.17771875, + "18": 418.6960234375, + "19": 419.20465625, + "20": 446.446828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.173984375, + "2": 115.3352109375, + "3": 144.679890625, + "4": 159.49659375, + "5": 174.3488828125, + "6": 198.7245703125, + "7": 223.7893046875, + "8": 243.8599140625, + "9": 258.4960625, + "11": 297.61453125, + "12": 307.7755703125, + "13": 325.891640625, + "14": 352.984046875, + "15": 372.9274609375, + "16": 377.635015625, + "17": 403.17771875, + "18": 418.6960234375, + "19": 419.20465625, + "20": 446.446828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 641473, + "answer_accuracy": 0.79220800071135, + "avg_response_len": 277.4719980042681, + "resp_tokens_sum": 224676846, + "pass_at_k": { + "pass@1": 0.79220800071135, + "pass@2": 0.8372970116454712, + "pass@4": 0.8704514423989349, + "pass@8": 0.8953483730802938, + "pass@16": 0.9141762790389022, + "pass@32": 0.9286124129103374, + "pass@64": 0.9404834726966399, + "pass@128": 0.9509958899778691 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9243869837758112, + "pass@2": 0.9460098599400741, + "pass@4": 0.9614304732926082, + "pass@8": 0.9726916923371863, + "pass@16": 0.9808099201760766, + "pass@32": 0.987371006759809, + "pass@64": 0.9923337148073603, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999026479750779, + "pass@2": 0.999998850172934, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985187308868502, + "pass@2": 0.9994356353391605, + "pass@4": 0.9999022283911758, + "pass@8": 0.9999973956511242, + "pass@16": 0.9999999990253551, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959228515625, + "pass@2": 0.9968446265994094, + "pass@4": 0.997471655593832, + "pass@8": 0.9980329679238036, + "pass@16": 0.9988023146599382, + "pass@32": 0.9996059543702204, + "pass@64": 0.9999794627424621, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9770337301587302, + "pass@2": 0.9836676665416823, + "pass@4": 0.9878900095821355, + "pass@8": 0.9906436267752851, + "pass@16": 0.9923964763989225, + "pass@32": 0.9940517762081893, + "pass@64": 0.9958489186730718, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9939740351973219, + "pass@4": 0.9941051227129544, + "pass@8": 0.9941991017897541, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9886048446745562, + "pass@2": 0.9934385628756462, + "pass@4": 0.9956759880791526, + "pass@8": 0.9967927655521467, + "pass@16": 0.9973605187841855, + "pass@32": 0.9977806459943003, + "pass@64": 0.998520710058505, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9702774439102564, + "pass@2": 0.982531104885928, + "pass@4": 0.9888667660532817, + "pass@8": 0.9931890969396343, + "pass@16": 0.9965075842527735, + "pass@32": 0.9988519859805879, + "pass@64": 0.9999112970726224, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9757482394366197, + "pass@2": 0.9878105245647109, + "pass@4": 0.9923439170455804, + "pass@8": 0.9945576813789561, + "pass@16": 0.9958275331654328, + "pass@32": 0.9966991001974037, + "pass@64": 0.9971441585191202, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9068918201133145, + "pass@2": 0.9340708020120008, + "pass@4": 0.9504890493504177, + "pass@8": 0.9617702037856002, + "pass@16": 0.9693008978007052, + "pass@32": 0.9742877622847005, + "pass@64": 0.9792593320688112, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.842867231638418, + "pass@2": 0.8940118416077222, + "pass@4": 0.9220305488826605, + "pass@8": 0.9400609534932638, + "pass@16": 0.9528386819412975, + "pass@32": 0.9611094486108223, + "pass@64": 0.9665597294441702, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.767682210031348, + "pass@2": 0.8322517617801694, + "pass@4": 0.8766375023067248, + "pass@8": 0.9080308723082444, + "pass@16": 0.9280418580337233, + "pass@32": 0.9415882651053201, + "pass@64": 0.9528324893138677, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.7715391355140186, + "pass@2": 0.8373182506684327, + "pass@4": 0.8764985632637043, + "pass@8": 0.9029473056185894, + "pass@16": 0.9240925109656627, + "pass@32": 0.9420440326754451, + "pass@64": 0.9570938658215324, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7029669436416185, + "pass@2": 0.7771679145236904, + "pass@4": 0.8295301861653124, + "pass@8": 0.868905281809576, + "pass@16": 0.897089399919766, + "pass@32": 0.9178947537440465, + "pass@64": 0.9354956627501524, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5763356854838709, + "pass@2": 0.6727306642113283, + "pass@4": 0.7455648006700768, + "pass@8": 0.798273290372829, + "pass@16": 0.8345066509257882, + "pass@32": 0.8602376954283026, + "pass@64": 0.8810953202102305, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5152637283236994, + "pass@2": 0.6144669393746302, + "pass@4": 0.6907101791105587, + "pass@8": 0.7469530410448263, + "pass@16": 0.7886757629108314, + "pass@32": 0.8232300018880613, + "pass@64": 0.8524341111586986, + "pass@128": 0.8757225433526011 + }, + "18": { + "pass@1": 0.4566071428571429, + "pass@2": 0.5588877952755906, + "pass@4": 0.6421446362954628, + "pass@8": 0.7082410808697225, + "pass@16": 0.7607307427371466, + "pass@32": 0.799389937452332, + "pass@64": 0.8314085165537201, + "pass@128": 0.8628571428571429 + }, + "19": { + "pass@1": 0.3484421921921922, + "pass@2": 0.44732589380719306, + "pass@4": 0.5323035952150127, + "pass@8": 0.6028325568129795, + "pass@16": 0.6648757853907019, + "pass@32": 0.7194209733459501, + "pass@64": 0.7657646008944604, + "pass@128": 0.8078078078078078 + }, + "20": { + "pass@1": 0.36751595345345345, + "pass@2": 0.4783597573951905, + "pass@4": 0.5728757984643807, + "pass@8": 0.6483896146476361, + "pass@16": 0.7054732379264461, + "pass@32": 0.7457587090598242, + "pass@64": 0.7768040635022134, + "pass@128": 0.8018018018018018 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 652442, + "answer_accuracy": 0.7945757014809042, + "avg_response_len": 286.8364611749805, + "resp_tokens_sum": 235527155, + "pass_at_k": { + "pass@1": 0.7945757014809042, + "pass@2": 0.8397753251176724, + "pass@4": 0.873430377150824, + "pass@8": 0.8985694516362919, + "pass@16": 0.9175146789996249, + "pass@32": 0.9321140964692218, + "pass@64": 0.9436201845387767, + "pass@128": 0.9526110678098207 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9198619631901841, + "pass@2": 0.9403108545480898, + "pass@4": 0.9548925379726305, + "pass@8": 0.966678628809453, + "pass@16": 0.9754899667090449, + "pass@32": 0.9819108547202062, + "pass@64": 0.9875499594029227, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974431818181818, + "pass@2": 0.9992409329515629, + "pass@4": 0.9998840809671518, + "pass@8": 0.9999964960477193, + "pass@16": 0.9999999981202536, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996995192307693, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9800271739130435, + "pass@2": 0.9857996690631067, + "pass@4": 0.9883860204974377, + "pass@8": 0.9901598054192082, + "pass@16": 0.9920850067141929, + "pass@32": 0.9943094397909279, + "pass@64": 0.996374879969186, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966048242447487, + "pass@4": 0.9971893615375443, + "pass@8": 0.9974785218231972, + "pass@16": 0.997808696404298, + "pass@32": 0.9983924824583172, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9900198699421965, + "pass@2": 0.9935380017978244, + "pass@4": 0.9945450686077535, + "pass@8": 0.995080130097074, + "pass@16": 0.9958600713338742, + "pass@32": 0.9971688871261034, + "pass@64": 0.9989304082654408, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9748679577464788, + "pass@2": 0.9831252772540756, + "pass@4": 0.9888548461371907, + "pass@8": 0.9932193457316243, + "pass@16": 0.9961788407990759, + "pass@32": 0.9981455597006466, + "pass@64": 0.9995750159591402, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9718843562874252, + "pass@2": 0.9816797261775662, + "pass@4": 0.9866750974990404, + "pass@8": 0.9905660602213172, + "pass@16": 0.9931415853062885, + "pass@32": 0.9939450571469223, + "pass@64": 0.9940117895736039, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9253812123493976, + "pass@2": 0.9544638939616732, + "pass@4": 0.9677863153965692, + "pass@8": 0.9736431669648126, + "pass@16": 0.9771981706043198, + "pass@32": 0.9804179898916575, + "pass@64": 0.9839934853283643, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8534972570532915, + "pass@2": 0.8976026985165256, + "pass@4": 0.9223919453250156, + "pass@8": 0.9368541385246163, + "pass@16": 0.9456598452786678, + "pass@32": 0.9524532517270197, + "pass@64": 0.9595871670233102, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7447478991596639, + "pass@2": 0.809078552680915, + "pass@4": 0.8497325329081762, + "pass@8": 0.8773586600126486, + "pass@16": 0.8976721335804148, + "pass@32": 0.9161543175079596, + "pass@64": 0.9345612772225022, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7345349049707602, + "pass@2": 0.8065829764700457, + "pass@4": 0.8545422951517018, + "pass@8": 0.8879746860407295, + "pass@16": 0.9123875941505499, + "pass@32": 0.9320443484017564, + "pass@64": 0.9469134412037941, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6902698863636364, + "pass@2": 0.7734207229778092, + "pass@4": 0.8322661962709201, + "pass@8": 0.8754337937292219, + "pass@16": 0.907189536730187, + "pass@32": 0.9293226717684716, + "pass@64": 0.9437996441836513, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5670405982905983, + "pass@2": 0.6678566721627744, + "pass@4": 0.7426079993205973, + "pass@8": 0.7965303456407977, + "pass@16": 0.8387066560797861, + "pass@32": 0.873853793076624, + "pass@64": 0.9024630188422406, + "pass@128": 0.9259259259259259 + }, + "17": { + "pass@1": 0.5341372282608695, + "pass@2": 0.6374235065046218, + "pass@4": 0.7168359166672484, + "pass@8": 0.7718156135361299, + "pass@16": 0.8099417862425061, + "pass@32": 0.8392414057749845, + "pass@64": 0.8639867822902568, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5121783088235294, + "pass@2": 0.617220067160723, + "pass@4": 0.7035296877412376, + "pass@8": 0.7725314289848646, + "pass@16": 0.8238449099525559, + "pass@32": 0.8584219170158941, + "pass@64": 0.8806343237101412, + "pass@128": 0.8941176470588236 + }, + "19": { + "pass@1": 0.34688895089285715, + "pass@2": 0.44558822237064094, + "pass@4": 0.5352641821000053, + "pass@8": 0.6142787717357525, + "pass@16": 0.6813574607373921, + "pass@32": 0.7350333718827479, + "pass@64": 0.7773131190777017, + "pass@128": 0.8125 + }, + "20": { + "pass@1": 0.34889595808383234, + "pass@2": 0.45785434544297254, + "pass@4": 0.5531497398902979, + "pass@8": 0.6277050654841096, + "pass@16": 0.6832874410114024, + "pass@32": 0.7244500613151685, + "pass@64": 0.7548202541371628, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 636376, + "answer_accuracy": 0.7943261703147467, + "avg_response_len": 307.5434012521968, + "resp_tokens_sum": 246389011, + "pass_at_k": { + "pass@1": 0.7943261703147467, + "pass@2": 0.8398748715235833, + "pass@4": 0.8735342392593926, + "pass@8": 0.8982648055791383, + "pass@16": 0.9164823078883126, + "pass@32": 0.9304510536012471, + "pass@64": 0.941697997129187, + "pass@128": 0.9511104010225275 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390158582089553, + "pass@2": 0.9639903337642501, + "pass@4": 0.9774934780540493, + "pass@8": 0.9848767093098696, + "pass@16": 0.9895625811845377, + "pass@32": 0.9926980858334635, + "pass@64": 0.9947788458509305, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9999776785714286, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996059584548105, + "pass@2": 0.9971562865866257, + "pass@4": 0.9975908537125278, + "pass@8": 0.9980364575309247, + "pass@16": 0.9987139656985613, + "pass@32": 0.9995016669456886, + "pass@64": 0.999959697670942, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974186769005848, + "pass@2": 0.9983095760233918, + "pass@4": 0.9990503037449264, + "pass@8": 0.9997044324501367, + "pass@16": 0.9999751571115222, + "pass@32": 0.9999999123006084, + "pass@64": 0.9999999999999893, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9751148897058823, + "pass@2": 0.9831331056044463, + "pass@4": 0.988671556772315, + "pass@8": 0.9929509867149238, + "pass@16": 0.996195633399387, + "pass@32": 0.998615426502192, + "pass@64": 0.9998703313410889, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.989870859621451, + "pass@2": 0.9927190193497106, + "pass@4": 0.994744807135701, + "pass@8": 0.9962839993861353, + "pass@16": 0.9975711019742235, + "pass@32": 0.9986572988061697, + "pass@64": 0.9996149764108252, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9795787183544303, + "pass@2": 0.9913235541463173, + "pass@4": 0.9965821076162948, + "pass@8": 0.9982044541228395, + "pass@16": 0.9990875747212965, + "pass@32": 0.9997847637200062, + "pass@64": 0.9999954280485364, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9837180930930931, + "pass@2": 0.9927041953370693, + "pass@4": 0.996670133294149, + "pass@8": 0.9987091388632008, + "pass@16": 0.9997070059982301, + "pass@32": 0.9999846478525416, + "pass@64": 0.9999999932642234, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9707345257234726, + "pass@2": 0.9843136820518011, + "pass@4": 0.9893559426454328, + "pass@8": 0.9921609070181736, + "pass@16": 0.9945186673490651, + "pass@32": 0.9966248494281174, + "pass@64": 0.9982949722027745, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9181299603174603, + "pass@2": 0.9498144763154606, + "pass@4": 0.9654031841257935, + "pass@8": 0.9746522564529527, + "pass@16": 0.9811809898411397, + "pass@32": 0.9857436588473356, + "pass@64": 0.9885079148127753, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8718941131498471, + "pass@2": 0.9195667485371669, + "pass@4": 0.948841029619004, + "pass@8": 0.9654747523656698, + "pass@16": 0.9751139020388929, + "pass@32": 0.9816078412869249, + "pass@64": 0.9874293943219232, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7740162037037037, + "pass@2": 0.8375000759453681, + "pass@4": 0.8756528162567636, + "pass@8": 0.8982326625655992, + "pass@16": 0.9136936012436747, + "pass@32": 0.927070669081338, + "pass@64": 0.9389615729467963, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7496986275964391, + "pass@2": 0.8206693643543079, + "pass@4": 0.8669634034061768, + "pass@8": 0.8957210779293493, + "pass@16": 0.9151217937643485, + "pass@32": 0.9305009446166331, + "pass@64": 0.9410892514052502, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6819540895061729, + "pass@2": 0.7613462379702535, + "pass@4": 0.8196656274447172, + "pass@8": 0.8599604416066629, + "pass@16": 0.8864494760423818, + "pass@32": 0.9045654658432458, + "pass@64": 0.9183223223885263, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.5715800147492626, + "pass@2": 0.6731016857129583, + "pass@4": 0.7461016927640676, + "pass@8": 0.7992945130458669, + "pass@16": 0.8390093434420521, + "pass@32": 0.869805682570058, + "pass@64": 0.8964608644548767, + "pass@128": 0.9233038348082596 + }, + "17": { + "pass@1": 0.5130600527108434, + "pass@2": 0.6144722838203205, + "pass@4": 0.6933464623397975, + "pass@8": 0.7538670399041609, + "pass@16": 0.8000821577644895, + "pass@32": 0.8343846127580472, + "pass@64": 0.8605566221506656, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.49553931451612904, + "pass@2": 0.5865177324104649, + "pass@4": 0.6576192465861119, + "pass@8": 0.7115264614347813, + "pass@16": 0.7503217430377445, + "pass@32": 0.7810964204600007, + "pass@64": 0.8083169134024887, + "pass@128": 0.832258064516129 + }, + "19": { + "pass@1": 0.33263311933534745, + "pass@2": 0.43333662904108294, + "pass@4": 0.5287418866825937, + "pass@8": 0.6103951658849588, + "pass@16": 0.6756559875944252, + "pass@32": 0.7286437508440294, + "pass@64": 0.7727997410431042, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3407939189189189, + "pass@2": 0.4491388534203494, + "pass@4": 0.5475690609056749, + "pass@8": 0.6306101189516876, + "pass@16": 0.6951740667469974, + "pass@32": 0.7435576624743155, + "pass@64": 0.7819581707237205, + "pass@128": 0.8168168168168168 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f60983857799e9b6a54872882d7484a2ef39dee2 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:785cda4c71a06420ae469149a6d6e2164cc1075f41ed1f639d54221beaf2fd22 +size 2705856525 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/summary.csv b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..1f47cd71a4d3a2d8fa4f79373d702b1609afe530 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op17-20,0.9809236111111111,0.62520859375,0.7937051809210526,0.0,0.0,0.0,199.76715711805556,372.23534921875,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/summary.json b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3dcd75022786f445541acd5e07fb152bbcf5dc12 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op17-20", + "id_acc": 0.9809236111111111, + "ood_acc": 0.62520859375, + "total_acc": 0.7937051809210526, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.76715711805556, + "ood_avg_resp_len": 372.23534921875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 641473, + "answer_accuracy": 0.79220800071135, + "avg_response_len": 277.4719980042681, + "resp_tokens_sum": 224676846, + "pass_at_k": { + "pass@1": 0.79220800071135, + "pass@2": 0.8372970116454712, + "pass@4": 0.8704514423989349, + "pass@8": 0.8953483730802938, + "pass@16": 0.9141762790389022, + "pass@32": 0.9286124129103374, + "pass@64": 0.9404834726966399, + "pass@128": 0.9509958899778691 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9243869837758112, + "pass@2": 0.9460098599400741, + "pass@4": 0.9614304732926082, + "pass@8": 0.9726916923371863, + "pass@16": 0.9808099201760766, + "pass@32": 0.987371006759809, + "pass@64": 0.9923337148073603, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999026479750779, + "pass@2": 0.999998850172934, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985187308868502, + "pass@2": 0.9994356353391605, + "pass@4": 0.9999022283911758, + "pass@8": 0.9999973956511242, + "pass@16": 0.9999999990253551, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959228515625, + "pass@2": 0.9968446265994094, + "pass@4": 0.997471655593832, + "pass@8": 0.9980329679238036, + "pass@16": 0.9988023146599382, + "pass@32": 0.9996059543702204, + "pass@64": 0.9999794627424621, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9770337301587302, + "pass@2": 0.9836676665416823, + "pass@4": 0.9878900095821355, + "pass@8": 0.9906436267752851, + "pass@16": 0.9923964763989225, + "pass@32": 0.9940517762081893, + "pass@64": 0.9958489186730718, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9939740351973219, + "pass@4": 0.9941051227129544, + "pass@8": 0.9941991017897541, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9886048446745562, + "pass@2": 0.9934385628756462, + "pass@4": 0.9956759880791526, + "pass@8": 0.9967927655521467, + "pass@16": 0.9973605187841855, + "pass@32": 0.9977806459943003, + "pass@64": 0.998520710058505, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9702774439102564, + "pass@2": 0.982531104885928, + "pass@4": 0.9888667660532817, + "pass@8": 0.9931890969396343, + "pass@16": 0.9965075842527735, + "pass@32": 0.9988519859805879, + "pass@64": 0.9999112970726224, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9757482394366197, + "pass@2": 0.9878105245647109, + "pass@4": 0.9923439170455804, + "pass@8": 0.9945576813789561, + "pass@16": 0.9958275331654328, + "pass@32": 0.9966991001974037, + "pass@64": 0.9971441585191202, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9068918201133145, + "pass@2": 0.9340708020120008, + "pass@4": 0.9504890493504177, + "pass@8": 0.9617702037856002, + "pass@16": 0.9693008978007052, + "pass@32": 0.9742877622847005, + "pass@64": 0.9792593320688112, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.842867231638418, + "pass@2": 0.8940118416077222, + "pass@4": 0.9220305488826605, + "pass@8": 0.9400609534932638, + "pass@16": 0.9528386819412975, + "pass@32": 0.9611094486108223, + "pass@64": 0.9665597294441702, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.767682210031348, + "pass@2": 0.8322517617801694, + "pass@4": 0.8766375023067248, + "pass@8": 0.9080308723082444, + "pass@16": 0.9280418580337233, + "pass@32": 0.9415882651053201, + "pass@64": 0.9528324893138677, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.7715391355140186, + "pass@2": 0.8373182506684327, + "pass@4": 0.8764985632637043, + "pass@8": 0.9029473056185894, + "pass@16": 0.9240925109656627, + "pass@32": 0.9420440326754451, + "pass@64": 0.9570938658215324, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7029669436416185, + "pass@2": 0.7771679145236904, + "pass@4": 0.8295301861653124, + "pass@8": 0.868905281809576, + "pass@16": 0.897089399919766, + "pass@32": 0.9178947537440465, + "pass@64": 0.9354956627501524, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5763356854838709, + "pass@2": 0.6727306642113283, + "pass@4": 0.7455648006700768, + "pass@8": 0.798273290372829, + "pass@16": 0.8345066509257882, + "pass@32": 0.8602376954283026, + "pass@64": 0.8810953202102305, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5152637283236994, + "pass@2": 0.6144669393746302, + "pass@4": 0.6907101791105587, + "pass@8": 0.7469530410448263, + "pass@16": 0.7886757629108314, + "pass@32": 0.8232300018880613, + "pass@64": 0.8524341111586986, + "pass@128": 0.8757225433526011 + }, + "18": { + "pass@1": 0.4566071428571429, + "pass@2": 0.5588877952755906, + "pass@4": 0.6421446362954628, + "pass@8": 0.7082410808697225, + "pass@16": 0.7607307427371466, + "pass@32": 0.799389937452332, + "pass@64": 0.8314085165537201, + "pass@128": 0.8628571428571429 + }, + "19": { + "pass@1": 0.3484421921921922, + "pass@2": 0.44732589380719306, + "pass@4": 0.5323035952150127, + "pass@8": 0.6028325568129795, + "pass@16": 0.6648757853907019, + "pass@32": 0.7194209733459501, + "pass@64": 0.7657646008944604, + "pass@128": 0.8078078078078078 + }, + "20": { + "pass@1": 0.36751595345345345, + "pass@2": 0.4783597573951905, + "pass@4": 0.5728757984643807, + "pass@8": 0.6483896146476361, + "pass@16": 0.7054732379264461, + "pass@32": 0.7457587090598242, + "pass@64": 0.7768040635022134, + "pass@128": 0.8018018018018018 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 652442, + "answer_accuracy": 0.7945757014809042, + "avg_response_len": 286.8364611749805, + "resp_tokens_sum": 235527155, + "pass_at_k": { + "pass@1": 0.7945757014809042, + "pass@2": 0.8397753251176724, + "pass@4": 0.873430377150824, + "pass@8": 0.8985694516362919, + "pass@16": 0.9175146789996249, + "pass@32": 0.9321140964692218, + "pass@64": 0.9436201845387767, + "pass@128": 0.9526110678098207 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9198619631901841, + "pass@2": 0.9403108545480898, + "pass@4": 0.9548925379726305, + "pass@8": 0.966678628809453, + "pass@16": 0.9754899667090449, + "pass@32": 0.9819108547202062, + "pass@64": 0.9875499594029227, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150812, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974431818181818, + "pass@2": 0.9992409329515629, + "pass@4": 0.9998840809671518, + "pass@8": 0.9999964960477193, + "pass@16": 0.9999999981202536, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996995192307693, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9800271739130435, + "pass@2": 0.9857996690631067, + "pass@4": 0.9883860204974377, + "pass@8": 0.9901598054192082, + "pass@16": 0.9920850067141929, + "pass@32": 0.9943094397909279, + "pass@64": 0.996374879969186, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966048242447487, + "pass@4": 0.9971893615375443, + "pass@8": 0.9974785218231972, + "pass@16": 0.997808696404298, + "pass@32": 0.9983924824583172, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9900198699421965, + "pass@2": 0.9935380017978244, + "pass@4": 0.9945450686077535, + "pass@8": 0.995080130097074, + "pass@16": 0.9958600713338742, + "pass@32": 0.9971688871261034, + "pass@64": 0.9989304082654408, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9748679577464788, + "pass@2": 0.9831252772540756, + "pass@4": 0.9888548461371907, + "pass@8": 0.9932193457316243, + "pass@16": 0.9961788407990759, + "pass@32": 0.9981455597006466, + "pass@64": 0.9995750159591402, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9718843562874252, + "pass@2": 0.9816797261775662, + "pass@4": 0.9866750974990404, + "pass@8": 0.9905660602213172, + "pass@16": 0.9931415853062885, + "pass@32": 0.9939450571469223, + "pass@64": 0.9940117895736039, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9253812123493976, + "pass@2": 0.9544638939616732, + "pass@4": 0.9677863153965692, + "pass@8": 0.9736431669648126, + "pass@16": 0.9771981706043198, + "pass@32": 0.9804179898916575, + "pass@64": 0.9839934853283643, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8534972570532915, + "pass@2": 0.8976026985165256, + "pass@4": 0.9223919453250156, + "pass@8": 0.9368541385246163, + "pass@16": 0.9456598452786678, + "pass@32": 0.9524532517270197, + "pass@64": 0.9595871670233102, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7447478991596639, + "pass@2": 0.809078552680915, + "pass@4": 0.8497325329081762, + "pass@8": 0.8773586600126486, + "pass@16": 0.8976721335804148, + "pass@32": 0.9161543175079596, + "pass@64": 0.9345612772225022, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7345349049707602, + "pass@2": 0.8065829764700457, + "pass@4": 0.8545422951517018, + "pass@8": 0.8879746860407295, + "pass@16": 0.9123875941505499, + "pass@32": 0.9320443484017564, + "pass@64": 0.9469134412037941, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6902698863636364, + "pass@2": 0.7734207229778092, + "pass@4": 0.8322661962709201, + "pass@8": 0.8754337937292219, + "pass@16": 0.907189536730187, + "pass@32": 0.9293226717684716, + "pass@64": 0.9437996441836513, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5670405982905983, + "pass@2": 0.6678566721627744, + "pass@4": 0.7426079993205973, + "pass@8": 0.7965303456407977, + "pass@16": 0.8387066560797861, + "pass@32": 0.873853793076624, + "pass@64": 0.9024630188422406, + "pass@128": 0.9259259259259259 + }, + "17": { + "pass@1": 0.5341372282608695, + "pass@2": 0.6374235065046218, + "pass@4": 0.7168359166672484, + "pass@8": 0.7718156135361299, + "pass@16": 0.8099417862425061, + "pass@32": 0.8392414057749845, + "pass@64": 0.8639867822902568, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5121783088235294, + "pass@2": 0.617220067160723, + "pass@4": 0.7035296877412376, + "pass@8": 0.7725314289848646, + "pass@16": 0.8238449099525559, + "pass@32": 0.8584219170158941, + "pass@64": 0.8806343237101412, + "pass@128": 0.8941176470588236 + }, + "19": { + "pass@1": 0.34688895089285715, + "pass@2": 0.44558822237064094, + "pass@4": 0.5352641821000053, + "pass@8": 0.6142787717357525, + "pass@16": 0.6813574607373921, + "pass@32": 0.7350333718827479, + "pass@64": 0.7773131190777017, + "pass@128": 0.8125 + }, + "20": { + "pass@1": 0.34889595808383234, + "pass@2": 0.45785434544297254, + "pass@4": 0.5531497398902979, + "pass@8": 0.6277050654841096, + "pass@16": 0.6832874410114024, + "pass@32": 0.7244500613151685, + "pass@64": 0.7548202541371628, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 636376, + "answer_accuracy": 0.7943261703147467, + "avg_response_len": 307.5434012521968, + "resp_tokens_sum": 246389011, + "pass_at_k": { + "pass@1": 0.7943261703147467, + "pass@2": 0.8398748715235833, + "pass@4": 0.8735342392593926, + "pass@8": 0.8982648055791383, + "pass@16": 0.9164823078883126, + "pass@32": 0.9304510536012471, + "pass@64": 0.941697997129187, + "pass@128": 0.9511104010225275 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390158582089553, + "pass@2": 0.9639903337642501, + "pass@4": 0.9774934780540493, + "pass@8": 0.9848767093098696, + "pass@16": 0.9895625811845377, + "pass@32": 0.9926980858334635, + "pass@64": 0.9947788458509305, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9999776785714286, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996059584548105, + "pass@2": 0.9971562865866257, + "pass@4": 0.9975908537125278, + "pass@8": 0.9980364575309247, + "pass@16": 0.9987139656985613, + "pass@32": 0.9995016669456886, + "pass@64": 0.999959697670942, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974186769005848, + "pass@2": 0.9983095760233918, + "pass@4": 0.9990503037449264, + "pass@8": 0.9997044324501367, + "pass@16": 0.9999751571115222, + "pass@32": 0.9999999123006084, + "pass@64": 0.9999999999999893, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9751148897058823, + "pass@2": 0.9831331056044463, + "pass@4": 0.988671556772315, + "pass@8": 0.9929509867149238, + "pass@16": 0.996195633399387, + "pass@32": 0.998615426502192, + "pass@64": 0.9998703313410889, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.989870859621451, + "pass@2": 0.9927190193497106, + "pass@4": 0.994744807135701, + "pass@8": 0.9962839993861353, + "pass@16": 0.9975711019742235, + "pass@32": 0.9986572988061697, + "pass@64": 0.9996149764108252, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9795787183544303, + "pass@2": 0.9913235541463173, + "pass@4": 0.9965821076162948, + "pass@8": 0.9982044541228395, + "pass@16": 0.9990875747212965, + "pass@32": 0.9997847637200062, + "pass@64": 0.9999954280485364, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9837180930930931, + "pass@2": 0.9927041953370693, + "pass@4": 0.996670133294149, + "pass@8": 0.9987091388632008, + "pass@16": 0.9997070059982301, + "pass@32": 0.9999846478525416, + "pass@64": 0.9999999932642234, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9707345257234726, + "pass@2": 0.9843136820518011, + "pass@4": 0.9893559426454328, + "pass@8": 0.9921609070181736, + "pass@16": 0.9945186673490651, + "pass@32": 0.9966248494281174, + "pass@64": 0.9982949722027745, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9181299603174603, + "pass@2": 0.9498144763154606, + "pass@4": 0.9654031841257935, + "pass@8": 0.9746522564529527, + "pass@16": 0.9811809898411397, + "pass@32": 0.9857436588473356, + "pass@64": 0.9885079148127753, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8718941131498471, + "pass@2": 0.9195667485371669, + "pass@4": 0.948841029619004, + "pass@8": 0.9654747523656698, + "pass@16": 0.9751139020388929, + "pass@32": 0.9816078412869249, + "pass@64": 0.9874293943219232, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7740162037037037, + "pass@2": 0.8375000759453681, + "pass@4": 0.8756528162567636, + "pass@8": 0.8982326625655992, + "pass@16": 0.9136936012436747, + "pass@32": 0.927070669081338, + "pass@64": 0.9389615729467963, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7496986275964391, + "pass@2": 0.8206693643543079, + "pass@4": 0.8669634034061768, + "pass@8": 0.8957210779293493, + "pass@16": 0.9151217937643485, + "pass@32": 0.9305009446166331, + "pass@64": 0.9410892514052502, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6819540895061729, + "pass@2": 0.7613462379702535, + "pass@4": 0.8196656274447172, + "pass@8": 0.8599604416066629, + "pass@16": 0.8864494760423818, + "pass@32": 0.9045654658432458, + "pass@64": 0.9183223223885263, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.5715800147492626, + "pass@2": 0.6731016857129583, + "pass@4": 0.7461016927640676, + "pass@8": 0.7992945130458669, + "pass@16": 0.8390093434420521, + "pass@32": 0.869805682570058, + "pass@64": 0.8964608644548767, + "pass@128": 0.9233038348082596 + }, + "17": { + "pass@1": 0.5130600527108434, + "pass@2": 0.6144722838203205, + "pass@4": 0.6933464623397975, + "pass@8": 0.7538670399041609, + "pass@16": 0.8000821577644895, + "pass@32": 0.8343846127580472, + "pass@64": 0.8605566221506656, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.49553931451612904, + "pass@2": 0.5865177324104649, + "pass@4": 0.6576192465861119, + "pass@8": 0.7115264614347813, + "pass@16": 0.7503217430377445, + "pass@32": 0.7810964204600007, + "pass@64": 0.8083169134024887, + "pass@128": 0.832258064516129 + }, + "19": { + "pass@1": 0.33263311933534745, + "pass@2": 0.43333662904108294, + "pass@4": 0.5287418866825937, + "pass@8": 0.6103951658849588, + "pass@16": 0.6756559875944252, + "pass@32": 0.7286437508440294, + "pass@64": 0.7727997410431042, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3407939189189189, + "pass@2": 0.4491388534203494, + "pass@4": 0.5475690609056749, + "pass@8": 0.6306101189516876, + "pass@16": 0.6951740667469974, + "pass@32": 0.7435576624743155, + "pass@64": 0.7819581707237205, + "pass@128": 0.8168168168168168 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8e09df29d1bd6e2cdbcddcb86bcadd91401a6571 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3965dfde154b15a18990ee3dbc1903bb11a2cd28f9300185247aaef10e38282 +size 1789702476 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..69e5879fbbde6233b6f343cc61c91ee1ac6de15f --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9881866319444444, + "avg_response_len": 200.56471788194443, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9881866319444444, + "pass@2": 0.9916940206692897, + "pass@4": 0.9937472080260802, + "pass@8": 0.9950256745433944, + "pass@16": 0.995918970147525, + "pass@32": 0.9967167007443143, + "pass@64": 0.9974373751506862, + "pass@128": 0.998 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.945546875, + "pass@2": 0.9619863435039367, + "pass@4": 0.9722789084176982, + "pass@8": 0.978320586747533, + "pass@16": 0.9821176094520658, + "pass@32": 0.9855344466300039, + "pass@64": 0.9886535044750449, + "pass@128": 0.991 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998828125, + "pass@2": 0.9999940944881889, + "pass@4": 0.9999999803149605, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998453125, + "pass@2": 0.9987360974409448, + "pass@4": 0.9989324409448819, + "pass@8": 0.9989959826806571, + "pass@16": 0.9989999908357302, + "pass@32": 0.9989999999999952, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9875078125, + "pass@2": 0.9902449557086617, + "pass@4": 0.9920332048181477, + "pass@8": 0.9935061300097088, + "pass@16": 0.9947781293611802, + "pass@32": 0.9959910107015573, + "pass@64": 0.9972239905843284, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.995921875, + "pass@2": 0.9970813238188977, + "pass@4": 0.9978752849643795, + "pass@8": 0.9983318928894995, + "pass@16": 0.9986815881195805, + "pass@32": 0.9990790701369168, + "pass@64": 0.9994861763011331, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9925, + "pass@2": 0.9951721210629921, + "pass@4": 0.9967957300337457, + "pass@8": 0.9979135128185609, + "pass@16": 0.9985177886161863, + "pass@32": 0.9988287194193279, + "pass@64": 0.998986176299525, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.988609375, + "pass@2": 0.9917342519685041, + "pass@4": 0.9933310793025871, + "pass@8": 0.9943687552948273, + "pass@16": 0.9952886427664974, + "pass@32": 0.9962078652371651, + "pass@64": 0.9971294772636621, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.9852578125, + "pass@2": 0.9902969980314964, + "pass@4": 0.9924782434383203, + "pass@8": 0.9937942104497948, + "pass@16": 0.9948869821764804, + "pass@32": 0.9958091945738614, + "pass@64": 0.9964570514324788, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.945546875, + "2": 1.0, + "3": 0.9998828125, + "4": 0.998453125, + "5": 0.9875078125, + "6": 0.995921875, + "7": 0.9925, + "8": 0.988609375, + "9": 0.9852578125 + }, + "per_op_avg_response_len": { + "10": 281.35696875, + "2": 115.778546875, + "3": 144.999046875, + "4": 160.322640625, + "5": 174.0378515625, + "6": 199.60325, + "7": 224.5355546875, + "8": 244.2178671875, + "9": 260.230734375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.35696875, + "2": 115.778546875, + "3": 144.999046875, + "4": 160.322640625, + "5": 174.0378515625, + "6": 199.60325, + "7": 224.5355546875, + "8": 244.2178671875, + "9": 260.230734375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 374152, + "answer_accuracy": 0.9871876055386694, + "avg_response_len": 191.13121886609252, + "resp_tokens_sum": 72440261, + "pass_at_k": { + "pass@1": 0.9871876055386694, + "pass@2": 0.9910065679954903, + "pass@4": 0.9933706134061002, + "pass@8": 0.9948787382528314, + "pass@16": 0.9959916049262881, + "pass@32": 0.996932009280768, + "pass@64": 0.997623874658365, + "pass@128": 0.9979736575481256 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9440910766961652, + "pass@2": 0.9619259256033263, + "pass@4": 0.9733008028310617, + "pass@8": 0.9796329614972789, + "pass@16": 0.9836198063396595, + "pass@32": 0.9871494149142288, + "pass@64": 0.9900664387317132, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997610856269113, + "pass@2": 0.9999830690601748, + "pass@4": 0.9999999398011028, + "pass@8": 0.9999999999999037, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9952392578125, + "pass@2": 0.9960503045029527, + "pass@4": 0.9966638779527559, + "pass@8": 0.9968624458770536, + "pass@16": 0.9968749713616567, + "pass@32": 0.9968749999999851, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9884448037745281, + "pass@4": 0.9912172847441687, + "pass@8": 0.9936405476960303, + "pass@16": 0.9955200953129588, + "pass@32": 0.9965242749238121, + "pass@64": 0.9968155681592348, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9952516841317365, + "pass@2": 0.9963617302795983, + "pass@4": 0.9969530315072892, + "pass@8": 0.9971869468609537, + "pass@16": 0.997380233015994, + "pass@32": 0.9977544910179635, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9939672707100592, + "pass@2": 0.9975007862367795, + "pass@4": 0.9989455513696882, + "pass@8": 0.9996740874998941, + "pass@16": 0.9999699583519792, + "pass@32": 0.9999998682386224, + "pass@64": 0.9999999999999689, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842497996794872, + "pass@2": 0.9867756915001012, + "pass@4": 0.9887697613519462, + "pass@8": 0.9910467989022552, + "pass@16": 0.9933610309010189, + "pass@32": 0.9952528489362457, + "pass@64": 0.9964022716657853, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9880941901408451, + "pass@2": 0.9925993955861153, + "pass@4": 0.9947421795867066, + "pass@8": 0.9960302302257878, + "pass@16": 0.9973287932023995, + "pass@32": 0.9988677095511874, + "pass@64": 0.9998798800098458, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386008, + "answer_accuracy": 0.9881020642201835, + "avg_response_len": 198.63679042431193, + "resp_tokens_sum": 77598654, + "pass_at_k": { + "pass@1": 0.9881020642201835, + "pass@2": 0.9909361422998736, + "pass@4": 0.9927346192432042, + "pass@8": 0.9939254172584381, + "pass@16": 0.994761320352493, + "pass@32": 0.9956205315037807, + "pass@64": 0.9965113107601999, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9321319018404908, + "pass@2": 0.9476168874692045, + "pass@4": 0.9591864146774595, + "pass@8": 0.9665758985161789, + "pass@16": 0.9713946408325, + "pass@32": 0.9765320586385488, + "pass@64": 0.9820343402756819, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9937726449275363, + "pass@2": 0.9958928877096884, + "pass@4": 0.996980766262913, + "pass@8": 0.997814488095137, + "pass@16": 0.9985277882902757, + "pass@32": 0.9993303332170732, + "pass@64": 0.9999164669633687, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971570558739254, + "pass@2": 0.9973976547165129, + "pass@4": 0.9976408598996759, + "pass@8": 0.9980702305191739, + "pass@16": 0.9987360751708015, + "pass@32": 0.9995102342761353, + "pass@64": 0.9999603905476594, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9961614884393064, + "pass@2": 0.9969082113922899, + "pass@4": 0.9970972621377531, + "pass@8": 0.9971097898864544, + "pass@16": 0.9971098265895272, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9906910211267606, + "pass@2": 0.9931511311966286, + "pass@4": 0.9941390846390681, + "pass@8": 0.9946572028223936, + "pass@16": 0.9950281835886983, + "pass@32": 0.995602750335372, + "pass@64": 0.9964844183209494, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9808897829341318, + "pass@2": 0.9857493428497335, + "pass@4": 0.988267169261028, + "pass@8": 0.9900623575143761, + "pass@16": 0.9911704025020784, + "pass@32": 0.9917586764732148, + "pass@64": 0.992514968870735, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378231, + "answer_accuracy": 0.9892633704385672, + "avg_response_len": 211.88598510210915, + "resp_tokens_sum": 81011640, + "pass_at_k": { + "pass@1": 0.9892633704385672, + "pass@2": 0.9931498600233566, + "pass@4": 0.9951551483123532, + "pass@8": 0.9962955317881365, + "pass@16": 0.997029808980642, + "pass@32": 0.9976232892765096, + "pass@64": 0.9981987154511626, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9600746268656717, + "pass@2": 0.9760308937595492, + "pass@4": 0.9839855673824355, + "pass@8": 0.9884217011453161, + "pass@16": 0.9910323647508269, + "pass@32": 0.9926607279340695, + "pass@64": 0.9936650352690197, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.999998923922775, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9831571691176471, + "pass@2": 0.986181753705419, + "pass@4": 0.987768811067734, + "pass@8": 0.9890098796842268, + "pass@16": 0.9902859187572153, + "pass@32": 0.9921085268843128, + "pass@64": 0.9948703103288452, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9952681388012619, + "pass@2": 0.9974912441938449, + "pass@4": 0.9991050862333061, + "pass@8": 0.9998263160465276, + "pass@16": 0.9999927446612897, + "pass@32": 0.999999994872384, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9869214794303798, + "pass@2": 0.9907804245988239, + "pass@4": 0.9941660790225588, + "pass@8": 0.9969103922243154, + "pass@16": 0.998506147763421, + "pass@32": 0.9994581137806127, + "pass@64": 0.9999562541124545, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9904748498498499, + "pass@2": 0.9948696235605684, + "pass@4": 0.9967433595012336, + "pass@8": 0.9971737148209429, + "pass@16": 0.9973723602143897, + "pass@32": 0.9977477477477459, + "pass@64": 0.9984984984984985, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9867112138263665, + "pass@2": 0.9925528362913638, + "pass@4": 0.9944165117423022, + "pass@8": 0.9952496826682898, + "pass@16": 0.9960911451894998, + "pass@32": 0.9966680377528495, + "pass@64": 0.9967835833638526, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.43237578125, + "avg_response_len": 331.06838125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.43237578125, + "pass@2": 0.49641370570865767, + "pass@4": 0.5495045835489272, + "pass@8": 0.5940228860196424, + "pass@16": 0.6320718716927703, + "pass@32": 0.6649014377054615, + "pass@64": 0.6936692087453568, + "pass@128": 0.7195 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.90653125, + "pass@2": 0.9345050442913394, + "pass@4": 0.9521217927446568, + "pass@8": 0.9642511992256196, + "pass@16": 0.9730952715275948, + "pass@32": 0.9798480818801598, + "pass@64": 0.9854767115424542, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8313828125, + "pass@2": 0.8823966535433072, + "pass@4": 0.9114985109673782, + "pass@8": 0.9305857758926962, + "pass@16": 0.944842607076872, + "pass@32": 0.9552371075302403, + "pass@64": 0.9619566990084746, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.652609375, + "pass@2": 0.7358287401574803, + "pass@4": 0.7913821734158243, + "pass@8": 0.8305039506299533, + "pass@16": 0.8598724396143157, + "pass@32": 0.8825472271928845, + "pass@64": 0.9013223030261772, + "pass@128": 0.919 + }, + "14": { + "pass@1": 0.5374921875, + "pass@2": 0.6274731791338588, + "pass@4": 0.6960043566741656, + "pass@8": 0.7486619914858712, + "pass@16": 0.7889511290367639, + "pass@32": 0.8190952128733852, + "pass@64": 0.8413761249989784, + "pass@128": 0.857 + }, + "15": { + "pass@1": 0.4008984375, + "pass@2": 0.4854404527559061, + "pass@4": 0.5576186936632918, + "pass@8": 0.6195653970378193, + "pass@16": 0.6724854102268251, + "pass@32": 0.7160075731678495, + "pass@64": 0.7500121940067608, + "pass@128": 0.776 + }, + "16": { + "pass@1": 0.264625, + "pass@2": 0.33291326279527556, + "pass@4": 0.39539860751780975, + "pass@8": 0.45183174704982376, + "pass@16": 0.5021316666654466, + "pass@32": 0.5473805066998924, + "pass@64": 0.5913889217172655, + "pass@128": 0.637 + }, + "17": { + "pass@1": 0.2116328125, + "pass@2": 0.27854109251968506, + "pass@4": 0.34129307395950487, + "pass@8": 0.3952669872639327, + "pass@16": 0.4423940361770327, + "pass@32": 0.48565735801481846, + "pass@64": 0.5265069420950885, + "pass@128": 0.567 + }, + "18": { + "pass@1": 0.2049609375, + "pass@2": 0.2619041584645669, + "pass@4": 0.31436011876640385, + "pass@8": 0.36157121222013106, + "pass@16": 0.40583861157095524, + "pass@32": 0.45032199288525837, + "pass@64": 0.49548496615742815, + "pass@128": 0.54 + }, + "19": { + "pass@1": 0.16628125, + "pass@2": 0.22363250492126, + "pass@4": 0.2791533695163101, + "pass@8": 0.32909221589257787, + "pass@16": 0.37264870267235944, + "pass@32": 0.4114749531303945, + "pass@64": 0.4472602581417296, + "pass@128": 0.48 + }, + "20": { + "pass@1": 0.14734375, + "pass@2": 0.20150196850393706, + "pass@4": 0.256215138263967, + "pass@8": 0.3088983834979792, + "pass@16": 0.3584588423594054, + "pass@32": 0.4014443636797277, + "pass@64": 0.43590696675916357, + "pass@128": 0.463 + } + }, + "per_op_accuracy": { + "11": 0.90653125, + "12": 0.8313828125, + "13": 0.652609375, + "14": 0.5374921875, + "15": 0.4008984375, + "16": 0.264625, + "17": 0.2116328125, + "18": 0.2049609375, + "19": 0.16628125, + "20": 0.14734375 + }, + "per_op_avg_response_len": { + "11": 297.9795078125, + "12": 307.738296875, + "13": 320.2966875, + "14": 333.7309921875, + "15": 341.566171875, + "16": 330.0242734375, + "17": 344.9595703125, + "18": 347.8157890625, + "19": 337.3244296875, + "20": 349.24809375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.9795078125, + "12": 307.738296875, + "13": 320.2966875, + "14": 333.7309921875, + "15": 341.566171875, + "16": 330.0242734375, + "17": 344.9595703125, + "18": 347.8157890625, + "19": 337.3244296875, + "20": 349.24809375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 184969, + "answer_accuracy": 0.42944140044576523, + "avg_response_len": 313.762474461367, + "resp_tokens_sum": 135143773, + "pass_at_k": { + "pass@1": 0.42944140044576523, + "pass@2": 0.49177261147055895, + "pass@4": 0.5428874922766899, + "pass@8": 0.5861685529208793, + "pass@16": 0.6248415168594406, + "pass@32": 0.6591573714285595, + "pass@64": 0.6894848924560293, + "pass@128": 0.7179791976225854 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9024876062322946, + "pass@2": 0.931581941067565, + "pass@4": 0.9496237440928949, + "pass@8": 0.9609451371143948, + "pass@16": 0.9687528799938224, + "pass@32": 0.97445366148869, + "pass@64": 0.980455894487472, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8017743644067796, + "pass@2": 0.8598980576315672, + "pass@4": 0.8944751257258097, + "pass@8": 0.9177371174751259, + "pass@16": 0.9360751130669388, + "pass@32": 0.9508038100292274, + "pass@64": 0.9607893326371673, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.659507249216301, + "pass@2": 0.7393949935823072, + "pass@4": 0.7958553585974161, + "pass@8": 0.8372123205283716, + "pass@16": 0.8674613523286514, + "pass@32": 0.8886753061248424, + "pass@64": 0.904222107718048, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5467776479750779, + "pass@2": 0.6388971931954772, + "pass@4": 0.7121483205370355, + "pass@8": 0.7704087737502552, + "pass@16": 0.815477537071034, + "pass@32": 0.847214306549488, + "pass@64": 0.8677718079301789, + "pass@128": 0.881619937694704 + }, + "15": { + "pass@1": 0.3922733020231214, + "pass@2": 0.4681888910381863, + "pass@4": 0.5306951159645505, + "pass@8": 0.5876536397160308, + "pass@16": 0.6422978102278127, + "pass@32": 0.6913246962419308, + "pass@64": 0.7320849649418045, + "pass@128": 0.7687861271676301 + }, + "16": { + "pass@1": 0.2746219758064516, + "pass@2": 0.34631858013716016, + "pass@4": 0.4080948180872554, + "pass@8": 0.4631322568491059, + "pass@16": 0.5165983115071396, + "pass@32": 0.5687127653893317, + "pass@64": 0.6185357796775787, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.19621567919075145, + "pass@2": 0.25674539734195084, + "pass@4": 0.3107480659028892, + "pass@8": 0.3572599305635648, + "pass@16": 0.401867191655092, + "pass@32": 0.4477144178921003, + "pass@64": 0.4946212787974844, + "pass@128": 0.5433526011560693 + }, + "18": { + "pass@1": 0.1870982142857143, + "pass@2": 0.24234392575928013, + "pass@4": 0.29290736916813953, + "pass@8": 0.33914655976527514, + "pass@16": 0.3828983728506247, + "pass@32": 0.42568057551467176, + "pass@64": 0.4696155969267014, + "pass@128": 0.5142857142857142 + }, + "19": { + "pass@1": 0.16356981981981983, + "pass@2": 0.22129449823839578, + "pass@4": 0.2778085534916244, + "pass@8": 0.3289818121334189, + "pass@16": 0.3757669922072508, + "pass@32": 0.4187583172258676, + "pass@64": 0.4592572719336854, + "pass@128": 0.4984984984984985 + }, + "20": { + "pass@1": 0.14433183183183182, + "pass@2": 0.19192802546641122, + "pass@4": 0.23964265165052556, + "pass@8": 0.28597395396564, + "pass@16": 0.33090534731059135, + "pass@32": 0.37000431446735793, + "pass@64": 0.4006310011626402, + "pass@128": 0.4264264264264264 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 183585, + "answer_accuracy": 0.4264816570026762, + "avg_response_len": 327.72537076271186, + "resp_tokens_sum": 141073974, + "pass_at_k": { + "pass@1": 0.4264816570026762, + "pass@2": 0.49038733666041684, + "pass@4": 0.5445636175495914, + "pass@8": 0.5903302308200826, + "pass@16": 0.6292431934076248, + "pass@32": 0.6629047681733515, + "pass@64": 0.692529264222625, + "pass@128": 0.7190008920606601 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9155685240963856, + "pass@2": 0.9410601312731242, + "pass@4": 0.9563942826988492, + "pass@8": 0.967432514276132, + "pass@16": 0.9755015021696759, + "pass@32": 0.9815980011599581, + "pass@64": 0.9860627617481424, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8446806426332288, + "pass@2": 0.8856316182459956, + "pass@4": 0.9081738398491723, + "pass@8": 0.9233247131091695, + "pass@16": 0.9350406606303716, + "pass@32": 0.944342086090132, + "pass@64": 0.9513388683872828, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6162902661064426, + "pass@2": 0.7046189538807642, + "pass@4": 0.7658860082090578, + "pass@8": 0.8102685794517571, + "pass@16": 0.8442982371096014, + "pass@32": 0.8709450621767988, + "pass@64": 0.893325649132335, + "pass@128": 0.9159663865546218 + }, + "14": { + "pass@1": 0.521358735380117, + "pass@2": 0.6097602828429342, + "pass@4": 0.6777620272794842, + "pass@8": 0.7293197545256065, + "pass@16": 0.7667814550018929, + "pass@32": 0.793833189433544, + "pass@64": 0.8168144632046505, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.3964962121212121, + "pass@2": 0.4862749045573844, + "pass@4": 0.5637376910272578, + "pass@8": 0.6297379981193137, + "pass@16": 0.68601350468554, + "pass@32": 0.7318830594705156, + "pass@64": 0.7661697400862987, + "pass@128": 0.7878787878787878 + }, + "16": { + "pass@1": 0.24988871082621084, + "pass@2": 0.3166483696749444, + "pass@4": 0.3774613835663703, + "pass@8": 0.43057208310926093, + "pass@16": 0.47661905874889815, + "pass@32": 0.5177668034936355, + "pass@64": 0.5574591442064127, + "pass@128": 0.5982905982905983 + }, + "17": { + "pass@1": 0.21705163043478262, + "pass@2": 0.2847514305277058, + "pass@4": 0.3514368285486051, + "pass@8": 0.41045573423135256, + "pass@16": 0.4616982211602544, + "pass@32": 0.5078805855261728, + "pass@64": 0.5510499585016837, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.20363051470588237, + "pass@2": 0.2591542670217693, + "pass@4": 0.3130542915591435, + "pass@8": 0.36448812855673524, + "pass@16": 0.4164891560243343, + "pass@32": 0.472010172075214, + "pass@64": 0.52766991652156, + "pass@128": 0.5794117647058824 + }, + "19": { + "pass@1": 0.17234002976190477, + "pass@2": 0.22890302774653173, + "pass@4": 0.2839505890446731, + "pass@8": 0.3321384213680405, + "pass@16": 0.37142437830858543, + "pass@32": 0.40509437843466223, + "pass@64": 0.43508699021666025, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.14050804640718562, + "pass@2": 0.1970330991560187, + "pass@4": 0.25565949484607836, + "pass@8": 0.31204667321026885, + "pass@16": 0.36406342565567207, + "pass@32": 0.40839658588372546, + "pass@64": 0.4440686098680625, + "pass@128": 0.47305389221556887 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 184887, + "answer_accuracy": 0.4414516159841076, + "avg_response_len": 352.30215894330684, + "resp_tokens_sum": 147549781, + "pass_at_k": { + "pass@1": 0.4414516159841076, + "pass@2": 0.5073806865217667, + "pass@4": 0.561388135134164, + "pass@8": 0.6058958170445341, + "pass@16": 0.6424150835164169, + "pass@32": 0.6728609678577314, + "pass@64": 0.6991441041437269, + "pass@128": 0.7215770171149144 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9015376984126984, + "pass@2": 0.9308719222597174, + "pass@4": 0.9504181245201488, + "pass@8": 0.9646030828081341, + "pass@16": 0.9754254164109293, + "pass@32": 0.9840488983795109, + "pass@64": 0.9904855361523591, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.850463493883792, + "pass@2": 0.9035971098509475, + "pass@4": 0.9331708304237193, + "pass@8": 0.9515787548155242, + "pass@16": 0.9638961660858965, + "pass@32": 0.9706649336915653, + "pass@64": 0.9735785206097072, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6858362268518519, + "pass@2": 0.7667060823126279, + "pass@4": 0.8150710157179423, + "pass@8": 0.8461954861639664, + "pass@16": 0.8695611036214453, + "pass@32": 0.8892975843887124, + "pass@64": 0.9072783763082582, + "pass@128": 0.9228395061728395 + }, + "14": { + "pass@1": 0.5450204005934718, + "pass@2": 0.6345672504030464, + "pass@4": 0.6991398589085861, + "pass@8": 0.7475769111699786, + "pass@16": 0.7861827359831308, + "pass@32": 0.817947981260319, + "pass@64": 0.8411597277074198, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.4145929783950617, + "pass@2": 0.5030135121998639, + "pass@4": 0.5801381033967972, + "pass@8": 0.6432830194959871, + "pass@16": 0.6909441708079473, + "pass@32": 0.7261969711199714, + "pass@64": 0.7526999441617819, + "pass@128": 0.7716049382716049 + }, + "16": { + "pass@1": 0.2707411504424779, + "pass@2": 0.3374953545629804, + "pass@4": 0.40236067338927756, + "pass@8": 0.4635101659446917, + "pass@16": 0.5153182904937165, + "pass@32": 0.5585351073832857, + "pass@64": 0.6016951929226099, + "pass@128": 0.6489675516224189 + }, + "17": { + "pass@1": 0.22244446536144577, + "pass@2": 0.2952326036429181, + "pass@4": 0.363287904712212, + "pass@8": 0.4201454965856751, + "pass@16": 0.4659071103938847, + "pass@32": 0.5036464484479651, + "pass@64": 0.5359334126313752, + "pass@128": 0.5632530120481928 + }, + "18": { + "pass@1": 0.22658770161290323, + "pass@2": 0.2870043021336043, + "pass@4": 0.3400131626691824, + "pass@8": 0.38369016965482305, + "pass@16": 0.42005763879020486, + "pass@32": 0.45435591274048576, + "pass@64": 0.489392695534684, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.16285876132930513, + "pass@2": 0.22063450055903128, + "pass@4": 0.2756366262374301, + "pass@8": 0.32611106608002355, + "pass@16": 0.3707543901983692, + "pass@32": 0.4106245387919458, + "pass@64": 0.44754790898795876, + "pass@128": 0.48036253776435045 + }, + "20": { + "pass@1": 0.1572118993993994, + "pass@2": 0.21555820091745284, + "pass@4": 0.2733449368941493, + "pass@8": 0.32866506899457026, + "pass@16": 0.3803909235314833, + "pass@32": 0.4259113131439447, + "pass@64": 0.46299677980802145, + "pass@128": 0.4894894894894895 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6956546052631579, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6956546052631579, + "pass@2": 0.7310201706899948, + "pass@4": 0.7599353004065103, + "pass@8": 0.7839715753203764, + "pass@16": 0.8044204972766043, + "pass@32": 0.8220770886186151, + "pass@64": 0.8375593928320908, + "pass@128": 0.851421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.945546875, + "pass@2": 0.9619863435039367, + "pass@4": 0.9722789084176982, + "pass@8": 0.978320586747533, + "pass@16": 0.9821176094520658, + "pass@32": 0.9855344466300039, + "pass@64": 0.9886535044750449, + "pass@128": 0.991 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998828125, + "pass@2": 0.9999940944881889, + "pass@4": 0.9999999803149605, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998453125, + "pass@2": 0.9987360974409448, + "pass@4": 0.9989324409448819, + "pass@8": 0.9989959826806571, + "pass@16": 0.9989999908357302, + "pass@32": 0.9989999999999952, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9875078125, + "pass@2": 0.9902449557086617, + "pass@4": 0.9920332048181477, + "pass@8": 0.9935061300097088, + "pass@16": 0.9947781293611802, + "pass@32": 0.9959910107015573, + "pass@64": 0.9972239905843284, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.995921875, + "pass@2": 0.9970813238188977, + "pass@4": 0.9978752849643795, + "pass@8": 0.9983318928894995, + "pass@16": 0.9986815881195805, + "pass@32": 0.9990790701369168, + "pass@64": 0.9994861763011331, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9925, + "pass@2": 0.9951721210629921, + "pass@4": 0.9967957300337457, + "pass@8": 0.9979135128185609, + "pass@16": 0.9985177886161863, + "pass@32": 0.9988287194193279, + "pass@64": 0.998986176299525, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.988609375, + "pass@2": 0.9917342519685041, + "pass@4": 0.9933310793025871, + "pass@8": 0.9943687552948273, + "pass@16": 0.9952886427664974, + "pass@32": 0.9962078652371651, + "pass@64": 0.9971294772636621, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.9852578125, + "pass@2": 0.9902969980314964, + "pass@4": 0.9924782434383203, + "pass@8": 0.9937942104497948, + "pass@16": 0.9948869821764804, + "pass@32": 0.9958091945738614, + "pass@64": 0.9964570514324788, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.90653125, + "pass@2": 0.9345050442913394, + "pass@4": 0.9521217927446568, + "pass@8": 0.9642511992256196, + "pass@16": 0.9730952715275948, + "pass@32": 0.9798480818801598, + "pass@64": 0.9854767115424542, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8313828125, + "pass@2": 0.8823966535433072, + "pass@4": 0.9114985109673782, + "pass@8": 0.9305857758926962, + "pass@16": 0.944842607076872, + "pass@32": 0.9552371075302403, + "pass@64": 0.9619566990084746, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.652609375, + "pass@2": 0.7358287401574803, + "pass@4": 0.7913821734158243, + "pass@8": 0.8305039506299533, + "pass@16": 0.8598724396143157, + "pass@32": 0.8825472271928845, + "pass@64": 0.9013223030261772, + "pass@128": 0.919 + }, + "14": { + "pass@1": 0.5374921875, + "pass@2": 0.6274731791338588, + "pass@4": 0.6960043566741656, + "pass@8": 0.7486619914858712, + "pass@16": 0.7889511290367639, + "pass@32": 0.8190952128733852, + "pass@64": 0.8413761249989784, + "pass@128": 0.857 + }, + "15": { + "pass@1": 0.4008984375, + "pass@2": 0.4854404527559061, + "pass@4": 0.5576186936632918, + "pass@8": 0.6195653970378193, + "pass@16": 0.6724854102268251, + "pass@32": 0.7160075731678495, + "pass@64": 0.7500121940067608, + "pass@128": 0.776 + }, + "16": { + "pass@1": 0.264625, + "pass@2": 0.33291326279527556, + "pass@4": 0.39539860751780975, + "pass@8": 0.45183174704982376, + "pass@16": 0.5021316666654466, + "pass@32": 0.5473805066998924, + "pass@64": 0.5913889217172655, + "pass@128": 0.637 + }, + "17": { + "pass@1": 0.2116328125, + "pass@2": 0.27854109251968506, + "pass@4": 0.34129307395950487, + "pass@8": 0.3952669872639327, + "pass@16": 0.4423940361770327, + "pass@32": 0.48565735801481846, + "pass@64": 0.5265069420950885, + "pass@128": 0.567 + }, + "18": { + "pass@1": 0.2049609375, + "pass@2": 0.2619041584645669, + "pass@4": 0.31436011876640385, + "pass@8": 0.36157121222013106, + "pass@16": 0.40583861157095524, + "pass@32": 0.45032199288525837, + "pass@64": 0.49548496615742815, + "pass@128": 0.54 + }, + "19": { + "pass@1": 0.16628125, + "pass@2": 0.22363250492126, + "pass@4": 0.2791533695163101, + "pass@8": 0.32909221589257787, + "pass@16": 0.37264870267235944, + "pass@32": 0.4114749531303945, + "pass@64": 0.4472602581417296, + "pass@128": 0.48 + }, + "20": { + "pass@1": 0.14734375, + "pass@2": 0.20150196850393706, + "pass@4": 0.256215138263967, + "pass@8": 0.3088983834979792, + "pass@16": 0.3584588423594054, + "pass@32": 0.4014443636797277, + "pass@64": 0.43590696675916357, + "pass@128": 0.463 + } + }, + "per_op_accuracy": { + "10": 0.945546875, + "2": 1.0, + "3": 0.9998828125, + "4": 0.998453125, + "5": 0.9875078125, + "6": 0.995921875, + "7": 0.9925, + "8": 0.988609375, + "9": 0.9852578125, + "11": 0.90653125, + "12": 0.8313828125, + "13": 0.652609375, + "14": 0.5374921875, + "15": 0.4008984375, + "16": 0.264625, + "17": 0.2116328125, + "18": 0.2049609375, + "19": 0.16628125, + "20": 0.14734375 + }, + "per_op_avg_response_len": { + "10": 281.35696875, + "2": 115.778546875, + "3": 144.999046875, + "4": 160.322640625, + "5": 174.0378515625, + "6": 199.60325, + "7": 224.5355546875, + "8": 244.2178671875, + "9": 260.230734375, + "11": 297.9795078125, + "12": 307.738296875, + "13": 320.2966875, + "14": 333.7309921875, + "15": 341.566171875, + "16": 330.0242734375, + "17": 344.9595703125, + "18": 347.8157890625, + "19": 337.3244296875, + "20": 349.24809375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.35696875, + "2": 115.778546875, + "3": 144.999046875, + "4": 160.322640625, + "5": 174.0378515625, + "6": 199.60325, + "7": 224.5355546875, + "8": 244.2178671875, + "9": 260.230734375, + "11": 297.9795078125, + "12": 307.738296875, + "13": 320.2966875, + "14": 333.7309921875, + "15": 341.566171875, + "16": 330.0242734375, + "17": 344.9595703125, + "18": 347.8157890625, + "19": 337.3244296875, + "20": 349.24809375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 559121, + "answer_accuracy": 0.6905047126936453, + "avg_response_len": 256.36267240357256, + "resp_tokens_sum": 207584034, + "pass_at_k": { + "pass@1": 0.6905047126936453, + "pass@2": 0.725448195610658, + "pass@4": 0.7537443562767125, + "pass@8": 0.7774728303106835, + "pass@16": 0.7985651037652235, + "pass@32": 0.8172589684377919, + "pass@64": 0.8337149788140985, + "pass@128": 0.8490357255769839 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9440910766961652, + "pass@2": 0.9619259256033263, + "pass@4": 0.9733008028310617, + "pass@8": 0.9796329614972789, + "pass@16": 0.9836198063396595, + "pass@32": 0.9871494149142288, + "pass@64": 0.9900664387317132, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997610856269113, + "pass@2": 0.9999830690601748, + "pass@4": 0.9999999398011028, + "pass@8": 0.9999999999999037, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9952392578125, + "pass@2": 0.9960503045029527, + "pass@4": 0.9966638779527559, + "pass@8": 0.9968624458770536, + "pass@16": 0.9968749713616567, + "pass@32": 0.9968749999999851, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9884448037745281, + "pass@4": 0.9912172847441687, + "pass@8": 0.9936405476960303, + "pass@16": 0.9955200953129588, + "pass@32": 0.9965242749238121, + "pass@64": 0.9968155681592348, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9952516841317365, + "pass@2": 0.9963617302795983, + "pass@4": 0.9969530315072892, + "pass@8": 0.9971869468609537, + "pass@16": 0.997380233015994, + "pass@32": 0.9977544910179635, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9939672707100592, + "pass@2": 0.9975007862367795, + "pass@4": 0.9989455513696882, + "pass@8": 0.9996740874998941, + "pass@16": 0.9999699583519792, + "pass@32": 0.9999998682386224, + "pass@64": 0.9999999999999689, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842497996794872, + "pass@2": 0.9867756915001012, + "pass@4": 0.9887697613519462, + "pass@8": 0.9910467989022552, + "pass@16": 0.9933610309010189, + "pass@32": 0.9952528489362457, + "pass@64": 0.9964022716657853, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9880941901408451, + "pass@2": 0.9925993955861153, + "pass@4": 0.9947421795867066, + "pass@8": 0.9960302302257878, + "pass@16": 0.9973287932023995, + "pass@32": 0.9988677095511874, + "pass@64": 0.9998798800098458, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9024876062322946, + "pass@2": 0.931581941067565, + "pass@4": 0.9496237440928949, + "pass@8": 0.9609451371143948, + "pass@16": 0.9687528799938224, + "pass@32": 0.97445366148869, + "pass@64": 0.980455894487472, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8017743644067796, + "pass@2": 0.8598980576315672, + "pass@4": 0.8944751257258097, + "pass@8": 0.9177371174751259, + "pass@16": 0.9360751130669388, + "pass@32": 0.9508038100292274, + "pass@64": 0.9607893326371673, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.659507249216301, + "pass@2": 0.7393949935823072, + "pass@4": 0.7958553585974161, + "pass@8": 0.8372123205283716, + "pass@16": 0.8674613523286514, + "pass@32": 0.8886753061248424, + "pass@64": 0.904222107718048, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5467776479750779, + "pass@2": 0.6388971931954772, + "pass@4": 0.7121483205370355, + "pass@8": 0.7704087737502552, + "pass@16": 0.815477537071034, + "pass@32": 0.847214306549488, + "pass@64": 0.8677718079301789, + "pass@128": 0.881619937694704 + }, + "15": { + "pass@1": 0.3922733020231214, + "pass@2": 0.4681888910381863, + "pass@4": 0.5306951159645505, + "pass@8": 0.5876536397160308, + "pass@16": 0.6422978102278127, + "pass@32": 0.6913246962419308, + "pass@64": 0.7320849649418045, + "pass@128": 0.7687861271676301 + }, + "16": { + "pass@1": 0.2746219758064516, + "pass@2": 0.34631858013716016, + "pass@4": 0.4080948180872554, + "pass@8": 0.4631322568491059, + "pass@16": 0.5165983115071396, + "pass@32": 0.5687127653893317, + "pass@64": 0.6185357796775787, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.19621567919075145, + "pass@2": 0.25674539734195084, + "pass@4": 0.3107480659028892, + "pass@8": 0.3572599305635648, + "pass@16": 0.401867191655092, + "pass@32": 0.4477144178921003, + "pass@64": 0.4946212787974844, + "pass@128": 0.5433526011560693 + }, + "18": { + "pass@1": 0.1870982142857143, + "pass@2": 0.24234392575928013, + "pass@4": 0.29290736916813953, + "pass@8": 0.33914655976527514, + "pass@16": 0.3828983728506247, + "pass@32": 0.42568057551467176, + "pass@64": 0.4696155969267014, + "pass@128": 0.5142857142857142 + }, + "19": { + "pass@1": 0.16356981981981983, + "pass@2": 0.22129449823839578, + "pass@4": 0.2778085534916244, + "pass@8": 0.3289818121334189, + "pass@16": 0.3757669922072508, + "pass@32": 0.4187583172258676, + "pass@64": 0.4592572719336854, + "pass@128": 0.4984984984984985 + }, + "20": { + "pass@1": 0.14433183183183182, + "pass@2": 0.19192802546641122, + "pass@4": 0.23964265165052556, + "pass@8": 0.28597395396564, + "pass@16": 0.33090534731059135, + "pass@32": 0.37000431446735793, + "pass@64": 0.4006310011626402, + "pass@128": 0.4264264264264264 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 569593, + "answer_accuracy": 0.6936781469212783, + "avg_response_len": 266.3101958300857, + "resp_tokens_sum": 218672628, + "pass_at_k": { + "pass@1": 0.6936781469212783, + "pass@2": 0.7285284052202895, + "pass@4": 0.7577854253701483, + "pass@8": 0.7823446515542763, + "pass@16": 0.8031420746914565, + "pass@32": 0.8211975989893279, + "pass@64": 0.8371517437288932, + "pass@128": 0.8514419329696025 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9321319018404908, + "pass@2": 0.9476168874692045, + "pass@4": 0.9591864146774595, + "pass@8": 0.9665758985161789, + "pass@16": 0.9713946408325, + "pass@32": 0.9765320586385488, + "pass@64": 0.9820343402756819, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9937726449275363, + "pass@2": 0.9958928877096884, + "pass@4": 0.996980766262913, + "pass@8": 0.997814488095137, + "pass@16": 0.9985277882902757, + "pass@32": 0.9993303332170732, + "pass@64": 0.9999164669633687, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971570558739254, + "pass@2": 0.9973976547165129, + "pass@4": 0.9976408598996759, + "pass@8": 0.9980702305191739, + "pass@16": 0.9987360751708015, + "pass@32": 0.9995102342761353, + "pass@64": 0.9999603905476594, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9961614884393064, + "pass@2": 0.9969082113922899, + "pass@4": 0.9970972621377531, + "pass@8": 0.9971097898864544, + "pass@16": 0.9971098265895272, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9906910211267606, + "pass@2": 0.9931511311966286, + "pass@4": 0.9941390846390681, + "pass@8": 0.9946572028223936, + "pass@16": 0.9950281835886983, + "pass@32": 0.995602750335372, + "pass@64": 0.9964844183209494, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9808897829341318, + "pass@2": 0.9857493428497335, + "pass@4": 0.988267169261028, + "pass@8": 0.9900623575143761, + "pass@16": 0.9911704025020784, + "pass@32": 0.9917586764732148, + "pass@64": 0.992514968870735, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9155685240963856, + "pass@2": 0.9410601312731242, + "pass@4": 0.9563942826988492, + "pass@8": 0.967432514276132, + "pass@16": 0.9755015021696759, + "pass@32": 0.9815980011599581, + "pass@64": 0.9860627617481424, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8446806426332288, + "pass@2": 0.8856316182459956, + "pass@4": 0.9081738398491723, + "pass@8": 0.9233247131091695, + "pass@16": 0.9350406606303716, + "pass@32": 0.944342086090132, + "pass@64": 0.9513388683872828, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6162902661064426, + "pass@2": 0.7046189538807642, + "pass@4": 0.7658860082090578, + "pass@8": 0.8102685794517571, + "pass@16": 0.8442982371096014, + "pass@32": 0.8709450621767988, + "pass@64": 0.893325649132335, + "pass@128": 0.9159663865546218 + }, + "14": { + "pass@1": 0.521358735380117, + "pass@2": 0.6097602828429342, + "pass@4": 0.6777620272794842, + "pass@8": 0.7293197545256065, + "pass@16": 0.7667814550018929, + "pass@32": 0.793833189433544, + "pass@64": 0.8168144632046505, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.3964962121212121, + "pass@2": 0.4862749045573844, + "pass@4": 0.5637376910272578, + "pass@8": 0.6297379981193137, + "pass@16": 0.68601350468554, + "pass@32": 0.7318830594705156, + "pass@64": 0.7661697400862987, + "pass@128": 0.7878787878787878 + }, + "16": { + "pass@1": 0.24988871082621084, + "pass@2": 0.3166483696749444, + "pass@4": 0.3774613835663703, + "pass@8": 0.43057208310926093, + "pass@16": 0.47661905874889815, + "pass@32": 0.5177668034936355, + "pass@64": 0.5574591442064127, + "pass@128": 0.5982905982905983 + }, + "17": { + "pass@1": 0.21705163043478262, + "pass@2": 0.2847514305277058, + "pass@4": 0.3514368285486051, + "pass@8": 0.41045573423135256, + "pass@16": 0.4616982211602544, + "pass@32": 0.5078805855261728, + "pass@64": 0.5510499585016837, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.20363051470588237, + "pass@2": 0.2591542670217693, + "pass@4": 0.3130542915591435, + "pass@8": 0.36448812855673524, + "pass@16": 0.4164891560243343, + "pass@32": 0.472010172075214, + "pass@64": 0.52766991652156, + "pass@128": 0.5794117647058824 + }, + "19": { + "pass@1": 0.17234002976190477, + "pass@2": 0.22890302774653173, + "pass@4": 0.2839505890446731, + "pass@8": 0.3321384213680405, + "pass@16": 0.37142437830858543, + "pass@32": 0.40509437843466223, + "pass@64": 0.43508699021666025, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.14050804640718562, + "pass@2": 0.1970330991560187, + "pass@4": 0.25565949484607836, + "pass@8": 0.31204667321026885, + "pass@16": 0.36406342565567207, + "pass@32": 0.40839658588372546, + "pass@64": 0.4440686098680625, + "pass@128": 0.47305389221556887 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 563118, + "answer_accuracy": 0.7028853451030516, + "avg_response_len": 285.290957271529, + "resp_tokens_sum": 228561421, + "pass_at_k": { + "pass@1": 0.7028853451030516, + "pass@2": 0.7392056619570142, + "pass@4": 0.7683959747831858, + "pass@8": 0.7922073600928059, + "pass@16": 0.8116488564772246, + "pass@32": 0.8278481949032498, + "pass@64": 0.8418627691022404, + "pass@128": 0.8538105128614795 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9600746268656717, + "pass@2": 0.9760308937595492, + "pass@4": 0.9839855673824355, + "pass@8": 0.9884217011453161, + "pass@16": 0.9910323647508269, + "pass@32": 0.9926607279340695, + "pass@64": 0.9936650352690197, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.999998923922775, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9831571691176471, + "pass@2": 0.986181753705419, + "pass@4": 0.987768811067734, + "pass@8": 0.9890098796842268, + "pass@16": 0.9902859187572153, + "pass@32": 0.9921085268843128, + "pass@64": 0.9948703103288452, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9952681388012619, + "pass@2": 0.9974912441938449, + "pass@4": 0.9991050862333061, + "pass@8": 0.9998263160465276, + "pass@16": 0.9999927446612897, + "pass@32": 0.999999994872384, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9869214794303798, + "pass@2": 0.9907804245988239, + "pass@4": 0.9941660790225588, + "pass@8": 0.9969103922243154, + "pass@16": 0.998506147763421, + "pass@32": 0.9994581137806127, + "pass@64": 0.9999562541124545, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9904748498498499, + "pass@2": 0.9948696235605684, + "pass@4": 0.9967433595012336, + "pass@8": 0.9971737148209429, + "pass@16": 0.9973723602143897, + "pass@32": 0.9977477477477459, + "pass@64": 0.9984984984984985, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9867112138263665, + "pass@2": 0.9925528362913638, + "pass@4": 0.9944165117423022, + "pass@8": 0.9952496826682898, + "pass@16": 0.9960911451894998, + "pass@32": 0.9966680377528495, + "pass@64": 0.9967835833638526, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9015376984126984, + "pass@2": 0.9308719222597174, + "pass@4": 0.9504181245201488, + "pass@8": 0.9646030828081341, + "pass@16": 0.9754254164109293, + "pass@32": 0.9840488983795109, + "pass@64": 0.9904855361523591, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.850463493883792, + "pass@2": 0.9035971098509475, + "pass@4": 0.9331708304237193, + "pass@8": 0.9515787548155242, + "pass@16": 0.9638961660858965, + "pass@32": 0.9706649336915653, + "pass@64": 0.9735785206097072, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6858362268518519, + "pass@2": 0.7667060823126279, + "pass@4": 0.8150710157179423, + "pass@8": 0.8461954861639664, + "pass@16": 0.8695611036214453, + "pass@32": 0.8892975843887124, + "pass@64": 0.9072783763082582, + "pass@128": 0.9228395061728395 + }, + "14": { + "pass@1": 0.5450204005934718, + "pass@2": 0.6345672504030464, + "pass@4": 0.6991398589085861, + "pass@8": 0.7475769111699786, + "pass@16": 0.7861827359831308, + "pass@32": 0.817947981260319, + "pass@64": 0.8411597277074198, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.4145929783950617, + "pass@2": 0.5030135121998639, + "pass@4": 0.5801381033967972, + "pass@8": 0.6432830194959871, + "pass@16": 0.6909441708079473, + "pass@32": 0.7261969711199714, + "pass@64": 0.7526999441617819, + "pass@128": 0.7716049382716049 + }, + "16": { + "pass@1": 0.2707411504424779, + "pass@2": 0.3374953545629804, + "pass@4": 0.40236067338927756, + "pass@8": 0.4635101659446917, + "pass@16": 0.5153182904937165, + "pass@32": 0.5585351073832857, + "pass@64": 0.6016951929226099, + "pass@128": 0.6489675516224189 + }, + "17": { + "pass@1": 0.22244446536144577, + "pass@2": 0.2952326036429181, + "pass@4": 0.363287904712212, + "pass@8": 0.4201454965856751, + "pass@16": 0.4659071103938847, + "pass@32": 0.5036464484479651, + "pass@64": 0.5359334126313752, + "pass@128": 0.5632530120481928 + }, + "18": { + "pass@1": 0.22658770161290323, + "pass@2": 0.2870043021336043, + "pass@4": 0.3400131626691824, + "pass@8": 0.38369016965482305, + "pass@16": 0.42005763879020486, + "pass@32": 0.45435591274048576, + "pass@64": 0.489392695534684, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.16285876132930513, + "pass@2": 0.22063450055903128, + "pass@4": 0.2756366262374301, + "pass@8": 0.32611106608002355, + "pass@16": 0.3707543901983692, + "pass@32": 0.4106245387919458, + "pass@64": 0.44754790898795876, + "pass@128": 0.48036253776435045 + }, + "20": { + "pass@1": 0.1572118993993994, + "pass@2": 0.21555820091745284, + "pass@4": 0.2733449368941493, + "pass@8": 0.32866506899457026, + "pass@16": 0.3803909235314833, + "pass@32": 0.4259113131439447, + "pass@64": 0.46299677980802145, + "pass@128": 0.4894894894894895 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1b1ae6a7fdf88f9b7cdcf638cd00ef20210d457f --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ba5e502173b4fc0f180e90fa43877fce26e361baec7babb5c871f8a80da335e +size 2576831414 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/summary.csv b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4c5569828e3b1859ffb007dc379487b417c6acd --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op7-10,0.9881866319444444,0.43237578125,0.6956546052631579,0.0,0.0,0.0,200.56471788194443,331.06838125,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/summary.json b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..65126984e432a9cea85147951a33591306487f4e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op7-10", + "id_acc": 0.9881866319444444, + "ood_acc": 0.43237578125, + "total_acc": 0.6956546052631579, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.56471788194443, + "ood_avg_resp_len": 331.06838125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 559121, + "answer_accuracy": 0.6905047126936453, + "avg_response_len": 256.36267240357256, + "resp_tokens_sum": 207584034, + "pass_at_k": { + "pass@1": 0.6905047126936453, + "pass@2": 0.725448195610658, + "pass@4": 0.7537443562767125, + "pass@8": 0.7774728303106835, + "pass@16": 0.7985651037652235, + "pass@32": 0.8172589684377919, + "pass@64": 0.8337149788140985, + "pass@128": 0.8490357255769839 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9440910766961652, + "pass@2": 0.9619259256033263, + "pass@4": 0.9733008028310617, + "pass@8": 0.9796329614972789, + "pass@16": 0.9836198063396595, + "pass@32": 0.9871494149142288, + "pass@64": 0.9900664387317132, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997610856269113, + "pass@2": 0.9999830690601748, + "pass@4": 0.9999999398011028, + "pass@8": 0.9999999999999037, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9952392578125, + "pass@2": 0.9960503045029527, + "pass@4": 0.9966638779527559, + "pass@8": 0.9968624458770536, + "pass@16": 0.9968749713616567, + "pass@32": 0.9968749999999851, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9884448037745281, + "pass@4": 0.9912172847441687, + "pass@8": 0.9936405476960303, + "pass@16": 0.9955200953129588, + "pass@32": 0.9965242749238121, + "pass@64": 0.9968155681592348, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9952516841317365, + "pass@2": 0.9963617302795983, + "pass@4": 0.9969530315072892, + "pass@8": 0.9971869468609537, + "pass@16": 0.997380233015994, + "pass@32": 0.9977544910179635, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9939672707100592, + "pass@2": 0.9975007862367795, + "pass@4": 0.9989455513696882, + "pass@8": 0.9996740874998941, + "pass@16": 0.9999699583519792, + "pass@32": 0.9999998682386224, + "pass@64": 0.9999999999999689, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842497996794872, + "pass@2": 0.9867756915001012, + "pass@4": 0.9887697613519462, + "pass@8": 0.9910467989022552, + "pass@16": 0.9933610309010189, + "pass@32": 0.9952528489362457, + "pass@64": 0.9964022716657853, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9880941901408451, + "pass@2": 0.9925993955861153, + "pass@4": 0.9947421795867066, + "pass@8": 0.9960302302257878, + "pass@16": 0.9973287932023995, + "pass@32": 0.9988677095511874, + "pass@64": 0.9998798800098458, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9024876062322946, + "pass@2": 0.931581941067565, + "pass@4": 0.9496237440928949, + "pass@8": 0.9609451371143948, + "pass@16": 0.9687528799938224, + "pass@32": 0.97445366148869, + "pass@64": 0.980455894487472, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8017743644067796, + "pass@2": 0.8598980576315672, + "pass@4": 0.8944751257258097, + "pass@8": 0.9177371174751259, + "pass@16": 0.9360751130669388, + "pass@32": 0.9508038100292274, + "pass@64": 0.9607893326371673, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.659507249216301, + "pass@2": 0.7393949935823072, + "pass@4": 0.7958553585974161, + "pass@8": 0.8372123205283716, + "pass@16": 0.8674613523286514, + "pass@32": 0.8886753061248424, + "pass@64": 0.904222107718048, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5467776479750779, + "pass@2": 0.6388971931954772, + "pass@4": 0.7121483205370355, + "pass@8": 0.7704087737502552, + "pass@16": 0.815477537071034, + "pass@32": 0.847214306549488, + "pass@64": 0.8677718079301789, + "pass@128": 0.881619937694704 + }, + "15": { + "pass@1": 0.3922733020231214, + "pass@2": 0.4681888910381863, + "pass@4": 0.5306951159645505, + "pass@8": 0.5876536397160308, + "pass@16": 0.6422978102278127, + "pass@32": 0.6913246962419308, + "pass@64": 0.7320849649418045, + "pass@128": 0.7687861271676301 + }, + "16": { + "pass@1": 0.2746219758064516, + "pass@2": 0.34631858013716016, + "pass@4": 0.4080948180872554, + "pass@8": 0.4631322568491059, + "pass@16": 0.5165983115071396, + "pass@32": 0.5687127653893317, + "pass@64": 0.6185357796775787, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.19621567919075145, + "pass@2": 0.25674539734195084, + "pass@4": 0.3107480659028892, + "pass@8": 0.3572599305635648, + "pass@16": 0.401867191655092, + "pass@32": 0.4477144178921003, + "pass@64": 0.4946212787974844, + "pass@128": 0.5433526011560693 + }, + "18": { + "pass@1": 0.1870982142857143, + "pass@2": 0.24234392575928013, + "pass@4": 0.29290736916813953, + "pass@8": 0.33914655976527514, + "pass@16": 0.3828983728506247, + "pass@32": 0.42568057551467176, + "pass@64": 0.4696155969267014, + "pass@128": 0.5142857142857142 + }, + "19": { + "pass@1": 0.16356981981981983, + "pass@2": 0.22129449823839578, + "pass@4": 0.2778085534916244, + "pass@8": 0.3289818121334189, + "pass@16": 0.3757669922072508, + "pass@32": 0.4187583172258676, + "pass@64": 0.4592572719336854, + "pass@128": 0.4984984984984985 + }, + "20": { + "pass@1": 0.14433183183183182, + "pass@2": 0.19192802546641122, + "pass@4": 0.23964265165052556, + "pass@8": 0.28597395396564, + "pass@16": 0.33090534731059135, + "pass@32": 0.37000431446735793, + "pass@64": 0.4006310011626402, + "pass@128": 0.4264264264264264 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 569593, + "answer_accuracy": 0.6936781469212783, + "avg_response_len": 266.3101958300857, + "resp_tokens_sum": 218672628, + "pass_at_k": { + "pass@1": 0.6936781469212783, + "pass@2": 0.7285284052202895, + "pass@4": 0.7577854253701483, + "pass@8": 0.7823446515542763, + "pass@16": 0.8031420746914565, + "pass@32": 0.8211975989893279, + "pass@64": 0.8371517437288932, + "pass@128": 0.8514419329696025 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9321319018404908, + "pass@2": 0.9476168874692045, + "pass@4": 0.9591864146774595, + "pass@8": 0.9665758985161789, + "pass@16": 0.9713946408325, + "pass@32": 0.9765320586385488, + "pass@64": 0.9820343402756819, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9937726449275363, + "pass@2": 0.9958928877096884, + "pass@4": 0.996980766262913, + "pass@8": 0.997814488095137, + "pass@16": 0.9985277882902757, + "pass@32": 0.9993303332170732, + "pass@64": 0.9999164669633687, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971570558739254, + "pass@2": 0.9973976547165129, + "pass@4": 0.9976408598996759, + "pass@8": 0.9980702305191739, + "pass@16": 0.9987360751708015, + "pass@32": 0.9995102342761353, + "pass@64": 0.9999603905476594, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9961614884393064, + "pass@2": 0.9969082113922899, + "pass@4": 0.9970972621377531, + "pass@8": 0.9971097898864544, + "pass@16": 0.9971098265895272, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9906910211267606, + "pass@2": 0.9931511311966286, + "pass@4": 0.9941390846390681, + "pass@8": 0.9946572028223936, + "pass@16": 0.9950281835886983, + "pass@32": 0.995602750335372, + "pass@64": 0.9964844183209494, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9808897829341318, + "pass@2": 0.9857493428497335, + "pass@4": 0.988267169261028, + "pass@8": 0.9900623575143761, + "pass@16": 0.9911704025020784, + "pass@32": 0.9917586764732148, + "pass@64": 0.992514968870735, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9155685240963856, + "pass@2": 0.9410601312731242, + "pass@4": 0.9563942826988492, + "pass@8": 0.967432514276132, + "pass@16": 0.9755015021696759, + "pass@32": 0.9815980011599581, + "pass@64": 0.9860627617481424, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8446806426332288, + "pass@2": 0.8856316182459956, + "pass@4": 0.9081738398491723, + "pass@8": 0.9233247131091695, + "pass@16": 0.9350406606303716, + "pass@32": 0.944342086090132, + "pass@64": 0.9513388683872828, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6162902661064426, + "pass@2": 0.7046189538807642, + "pass@4": 0.7658860082090578, + "pass@8": 0.8102685794517571, + "pass@16": 0.8442982371096014, + "pass@32": 0.8709450621767988, + "pass@64": 0.893325649132335, + "pass@128": 0.9159663865546218 + }, + "14": { + "pass@1": 0.521358735380117, + "pass@2": 0.6097602828429342, + "pass@4": 0.6777620272794842, + "pass@8": 0.7293197545256065, + "pass@16": 0.7667814550018929, + "pass@32": 0.793833189433544, + "pass@64": 0.8168144632046505, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.3964962121212121, + "pass@2": 0.4862749045573844, + "pass@4": 0.5637376910272578, + "pass@8": 0.6297379981193137, + "pass@16": 0.68601350468554, + "pass@32": 0.7318830594705156, + "pass@64": 0.7661697400862987, + "pass@128": 0.7878787878787878 + }, + "16": { + "pass@1": 0.24988871082621084, + "pass@2": 0.3166483696749444, + "pass@4": 0.3774613835663703, + "pass@8": 0.43057208310926093, + "pass@16": 0.47661905874889815, + "pass@32": 0.5177668034936355, + "pass@64": 0.5574591442064127, + "pass@128": 0.5982905982905983 + }, + "17": { + "pass@1": 0.21705163043478262, + "pass@2": 0.2847514305277058, + "pass@4": 0.3514368285486051, + "pass@8": 0.41045573423135256, + "pass@16": 0.4616982211602544, + "pass@32": 0.5078805855261728, + "pass@64": 0.5510499585016837, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.20363051470588237, + "pass@2": 0.2591542670217693, + "pass@4": 0.3130542915591435, + "pass@8": 0.36448812855673524, + "pass@16": 0.4164891560243343, + "pass@32": 0.472010172075214, + "pass@64": 0.52766991652156, + "pass@128": 0.5794117647058824 + }, + "19": { + "pass@1": 0.17234002976190477, + "pass@2": 0.22890302774653173, + "pass@4": 0.2839505890446731, + "pass@8": 0.3321384213680405, + "pass@16": 0.37142437830858543, + "pass@32": 0.40509437843466223, + "pass@64": 0.43508699021666025, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.14050804640718562, + "pass@2": 0.1970330991560187, + "pass@4": 0.25565949484607836, + "pass@8": 0.31204667321026885, + "pass@16": 0.36406342565567207, + "pass@32": 0.40839658588372546, + "pass@64": 0.4440686098680625, + "pass@128": 0.47305389221556887 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 563118, + "answer_accuracy": 0.7028853451030516, + "avg_response_len": 285.290957271529, + "resp_tokens_sum": 228561421, + "pass_at_k": { + "pass@1": 0.7028853451030516, + "pass@2": 0.7392056619570142, + "pass@4": 0.7683959747831858, + "pass@8": 0.7922073600928059, + "pass@16": 0.8116488564772246, + "pass@32": 0.8278481949032498, + "pass@64": 0.8418627691022404, + "pass@128": 0.8538105128614795 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9600746268656717, + "pass@2": 0.9760308937595492, + "pass@4": 0.9839855673824355, + "pass@8": 0.9884217011453161, + "pass@16": 0.9910323647508269, + "pass@32": 0.9926607279340695, + "pass@64": 0.9936650352690197, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.999998923922775, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9831571691176471, + "pass@2": 0.986181753705419, + "pass@4": 0.987768811067734, + "pass@8": 0.9890098796842268, + "pass@16": 0.9902859187572153, + "pass@32": 0.9921085268843128, + "pass@64": 0.9948703103288452, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9952681388012619, + "pass@2": 0.9974912441938449, + "pass@4": 0.9991050862333061, + "pass@8": 0.9998263160465276, + "pass@16": 0.9999927446612897, + "pass@32": 0.999999994872384, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9869214794303798, + "pass@2": 0.9907804245988239, + "pass@4": 0.9941660790225588, + "pass@8": 0.9969103922243154, + "pass@16": 0.998506147763421, + "pass@32": 0.9994581137806127, + "pass@64": 0.9999562541124545, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9904748498498499, + "pass@2": 0.9948696235605684, + "pass@4": 0.9967433595012336, + "pass@8": 0.9971737148209429, + "pass@16": 0.9973723602143897, + "pass@32": 0.9977477477477459, + "pass@64": 0.9984984984984985, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9867112138263665, + "pass@2": 0.9925528362913638, + "pass@4": 0.9944165117423022, + "pass@8": 0.9952496826682898, + "pass@16": 0.9960911451894998, + "pass@32": 0.9966680377528495, + "pass@64": 0.9967835833638526, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9015376984126984, + "pass@2": 0.9308719222597174, + "pass@4": 0.9504181245201488, + "pass@8": 0.9646030828081341, + "pass@16": 0.9754254164109293, + "pass@32": 0.9840488983795109, + "pass@64": 0.9904855361523591, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.850463493883792, + "pass@2": 0.9035971098509475, + "pass@4": 0.9331708304237193, + "pass@8": 0.9515787548155242, + "pass@16": 0.9638961660858965, + "pass@32": 0.9706649336915653, + "pass@64": 0.9735785206097072, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6858362268518519, + "pass@2": 0.7667060823126279, + "pass@4": 0.8150710157179423, + "pass@8": 0.8461954861639664, + "pass@16": 0.8695611036214453, + "pass@32": 0.8892975843887124, + "pass@64": 0.9072783763082582, + "pass@128": 0.9228395061728395 + }, + "14": { + "pass@1": 0.5450204005934718, + "pass@2": 0.6345672504030464, + "pass@4": 0.6991398589085861, + "pass@8": 0.7475769111699786, + "pass@16": 0.7861827359831308, + "pass@32": 0.817947981260319, + "pass@64": 0.8411597277074198, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.4145929783950617, + "pass@2": 0.5030135121998639, + "pass@4": 0.5801381033967972, + "pass@8": 0.6432830194959871, + "pass@16": 0.6909441708079473, + "pass@32": 0.7261969711199714, + "pass@64": 0.7526999441617819, + "pass@128": 0.7716049382716049 + }, + "16": { + "pass@1": 0.2707411504424779, + "pass@2": 0.3374953545629804, + "pass@4": 0.40236067338927756, + "pass@8": 0.4635101659446917, + "pass@16": 0.5153182904937165, + "pass@32": 0.5585351073832857, + "pass@64": 0.6016951929226099, + "pass@128": 0.6489675516224189 + }, + "17": { + "pass@1": 0.22244446536144577, + "pass@2": 0.2952326036429181, + "pass@4": 0.363287904712212, + "pass@8": 0.4201454965856751, + "pass@16": 0.4659071103938847, + "pass@32": 0.5036464484479651, + "pass@64": 0.5359334126313752, + "pass@128": 0.5632530120481928 + }, + "18": { + "pass@1": 0.22658770161290323, + "pass@2": 0.2870043021336043, + "pass@4": 0.3400131626691824, + "pass@8": 0.38369016965482305, + "pass@16": 0.42005763879020486, + "pass@32": 0.45435591274048576, + "pass@64": 0.489392695534684, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.16285876132930513, + "pass@2": 0.22063450055903128, + "pass@4": 0.2756366262374301, + "pass@8": 0.32611106608002355, + "pass@16": 0.3707543901983692, + "pass@32": 0.4106245387919458, + "pass@64": 0.44754790898795876, + "pass@128": 0.48036253776435045 + }, + "20": { + "pass@1": 0.1572118993993994, + "pass@2": 0.21555820091745284, + "pass@4": 0.2733449368941493, + "pass@8": 0.32866506899457026, + "pass@16": 0.3803909235314833, + "pass@32": 0.4259113131439447, + "pass@64": 0.46299677980802145, + "pass@128": 0.4894894894894895 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..58fa639f46eb4e5547387a804ee828b8f408fe00 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d3d7b69e8bb3cfa57a071b347c08d89e9c28c96d552e49b9d743835666e05fa +size 1790358289 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f8e2dc3b65e5b85a1f6672061800c3e3070e26e6 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9864262152777777, + "avg_response_len": 200.23685329861112, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9864262152777777, + "pass@2": 0.9902442585301815, + "pass@4": 0.9925803712973383, + "pass@8": 0.9943613064558152, + "pass@16": 0.9958117866313767, + "pass@32": 0.9969683649996883, + "pass@64": 0.9978454942095365, + "pass@128": 0.9984444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.94303125, + "pass@2": 0.9573801673228353, + "pass@4": 0.9675857014435695, + "pass@8": 0.9757808435253364, + "pass@16": 0.9823870776171006, + "pass@32": 0.9876964807576623, + "pass@64": 0.9919439701585595, + "pass@128": 0.995 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999765625, + "pass@2": 0.9999808070866142, + "pass@4": 0.9999997759655043, + "pass@8": 0.9999999999829964, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9970859375, + "pass@2": 0.9977909694881889, + "pass@4": 0.9981468881702288, + "pass@8": 0.9983682961001835, + "pass@16": 0.9986167377672557, + "pass@32": 0.9988739053984702, + "pass@64": 0.9989934280775878, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.984890625, + "pass@2": 0.9885019685039373, + "pass@4": 0.9909092400637421, + "pass@8": 0.9931073260534368, + "pass@16": 0.9949935107762257, + "pass@32": 0.9963786095513063, + "pass@64": 0.9974462093451094, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9942109375, + "pass@2": 0.9950751722440945, + "pass@4": 0.9955810483689539, + "pass@8": 0.9960776518199143, + "pass@16": 0.9966914915112858, + "pass@32": 0.9972938163882501, + "pass@64": 0.9977453345301317, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9901796875, + "pass@2": 0.9934657972440946, + "pass@4": 0.9952312505624299, + "pass@8": 0.9963188584246357, + "pass@16": 0.9972675226811877, + "pass@32": 0.9981704078751042, + "pass@64": 0.9987416329818158, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.985828125, + "pass@2": 0.990656126968504, + "pass@4": 0.9932887762467191, + "pass@8": 0.9950519033029102, + "pass@16": 0.9964262465725706, + "pass@32": 0.9975526463198227, + "pass@64": 0.9983623777967937, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9828515625, + "pass@2": 0.989347317913386, + "pass@4": 0.9924806608548928, + "pass@8": 0.9945468788929211, + "pass@16": 0.9959234927567568, + "pass@32": 0.9967494187065778, + "pass@64": 0.9973764949958263, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.94303125, + "2": 0.9999921875, + "3": 0.999765625, + "4": 0.9970859375, + "5": 0.984890625, + "6": 0.9942109375, + "7": 0.9901796875, + "8": 0.985828125, + "9": 0.9828515625 + }, + "per_op_avg_response_len": { + "10": 280.6092265625, + "2": 115.6311328125, + "3": 144.821484375, + "4": 160.1008984375, + "5": 174.0573125, + "6": 199.284875, + "7": 224.164765625, + "8": 243.8772109375, + "9": 259.5847734375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.6092265625, + "2": 115.6311328125, + "3": 144.821484375, + "4": 160.1008984375, + "5": 174.0573125, + "6": 199.284875, + "7": 224.164765625, + "8": 243.8772109375, + "9": 259.5847734375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373644, + "answer_accuracy": 0.98584726443769, + "avg_response_len": 190.77722897669707, + "resp_tokens_sum": 72306096, + "pass_at_k": { + "pass@1": 0.98584726443769, + "pass@2": 0.9899068467505395, + "pass@4": 0.992449547118414, + "pass@8": 0.9942836576845538, + "pass@16": 0.9956236387602456, + "pass@32": 0.9967898208722146, + "pass@64": 0.9978495753637363, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412103613569321, + "pass@2": 0.9582161086335446, + "pass@4": 0.9704242616796793, + "pass@8": 0.978816878970928, + "pass@16": 0.9845971539341197, + "pass@32": 0.9898851916877327, + "pass@64": 0.9949850127164509, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998566513761468, + "pass@2": 0.9999962375689279, + "pass@4": 0.999999998566693, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.9940810316190944, + "pass@4": 0.9943864211309524, + "pass@8": 0.9949083553134181, + "pass@16": 0.9956773146738891, + "pass@32": 0.9964809543702204, + "pass@64": 0.996854462742462, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.985218253968254, + "pass@2": 0.9877667635295586, + "pass@4": 0.9895235125371236, + "pass@8": 0.990932228795399, + "pass@16": 0.9919952093986906, + "pass@32": 0.992917042399551, + "pass@64": 0.9935593050866022, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9928892215568862, + "pass@2": 0.9938966800650667, + "pass@4": 0.9941838760611511, + "pass@8": 0.9943758755054594, + "pass@16": 0.994716276109128, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9916327662721893, + "pass@2": 0.9952126916088152, + "pass@4": 0.9968094700070774, + "pass@8": 0.9973776120848545, + "pass@16": 0.9977373270741638, + "pass@32": 0.9983401667986244, + "pass@64": 0.9992661790057308, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9821464342948718, + "pass@2": 0.9871637139107611, + "pass@4": 0.9906239857037101, + "pass@8": 0.9937512123492258, + "pass@16": 0.9961718325495631, + "pass@32": 0.998099042572955, + "pass@64": 0.9995645131644167, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9873239436619718, + "pass@2": 0.9934907674392813, + "pass@4": 0.9963719001674085, + "pass@8": 0.9984650960718653, + "pass@16": 0.999695388738615, + "pass@32": 0.9999885966972071, + "pass@64": 0.9999999973129757, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385635, + "answer_accuracy": 0.9871472599934469, + "avg_response_len": 198.27236494511794, + "resp_tokens_sum": 77456289, + "pass_at_k": { + "pass@1": 0.9871472599934469, + "pass@2": 0.9902974830626105, + "pass@4": 0.9922825805031253, + "pass@8": 0.9939972932211297, + "pass@16": 0.9954781164719507, + "pass@32": 0.9965650678593373, + "pass@64": 0.9974150587047373, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9334499616564417, + "pass@2": 0.9457872657118013, + "pass@4": 0.9571576134463252, + "pass@8": 0.9686398110946103, + "pass@16": 0.9785876056618497, + "pass@32": 0.9856178979361547, + "pass@64": 0.9907399312937869, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998579545454546, + "pass@2": 0.9999962717728466, + "pass@4": 0.9999999985797231, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981971153846154, + "pass@2": 0.999273458975912, + "pass@4": 0.9998449130619028, + "pass@8": 0.9999930032520015, + "pass@16": 0.9999999913361977, + "pass@32": 0.999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991304347826087, + "pass@2": 0.9943758558712769, + "pass@4": 0.9953860142482189, + "pass@8": 0.9962545915555301, + "pass@16": 0.9973181192300502, + "pass@32": 0.998318582579071, + "pass@64": 0.999280888268243, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968660458452722, + "pass@2": 0.9971247997653588, + "pass@4": 0.9971346516857312, + "pass@8": 0.997134670487104, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9948292991329479, + "pass@2": 0.996495024691639, + "pass@4": 0.9970465990342248, + "pass@8": 0.9971089117505242, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9894586267605634, + "pass@2": 0.9937756460019963, + "pass@4": 0.9952096093622099, + "pass@8": 0.9960140820363009, + "pass@16": 0.9969277354949511, + "pass@32": 0.9977852387074259, + "pass@64": 0.9985906885243891, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9783168038922155, + "pass@2": 0.9840545287378, + "pass@4": 0.9873645827018628, + "pass@8": 0.9898593075438438, + "pass@16": 0.9916034663346961, + "pass@32": 0.9927289869718435, + "pass@64": 0.993646529001145, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 377084, + "answer_accuracy": 0.9862633913625711, + "avg_response_len": 211.62137491630398, + "resp_tokens_sum": 80910470, + "pass_at_k": { + "pass@1": 0.9862633913625711, + "pass@2": 0.9905243505979989, + "pass@4": 0.9930143277411723, + "pass@8": 0.9948102138558693, + "pass@16": 0.996339227934685, + "pass@32": 0.9975574282851916, + "pass@64": 0.9982812507756754, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9541977611940299, + "pass@2": 0.9678156951463157, + "pass@4": 0.9748611783601676, + "pass@8": 0.9796577407085042, + "pass@16": 0.9838480089184216, + "pass@32": 0.9875043762636851, + "pass@64": 0.9900383081967405, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999512178324649, + "pass@4": 0.9999993495710995, + "pass@8": 0.9999999999504271, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999109100877193, + "pass@2": 0.9997971059538611, + "pass@4": 0.9999872888695931, + "pass@8": 0.9999999628675824, + "pass@16": 0.999999999999931, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.978079044117647, + "pass@2": 0.9832228462251044, + "pass@4": 0.9876504667615077, + "pass@8": 0.9919289408712592, + "pass@16": 0.9954125549449143, + "pass@32": 0.9976171476637306, + "pass@64": 0.9991856523243703, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9926804022082019, + "pass@2": 0.9940603343351797, + "pass@4": 0.9953427139628053, + "pass@8": 0.9967069697195312, + "pass@16": 0.998284717005795, + "pass@32": 0.9995420820514109, + "pass@64": 0.9999790726378381, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.983534414556962, + "pass@2": 0.9882804713196452, + "pass@4": 0.9915554633993535, + "pass@8": 0.9943213356780815, + "pass@16": 0.9969376777404727, + "pass@32": 0.9991500996745862, + "pass@64": 0.9999672926515153, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9854072822822822, + "pass@2": 0.9906026843772905, + "pass@4": 0.9937377819326637, + "pass@8": 0.9952448226037982, + "pass@16": 0.9961299961453437, + "pass@32": 0.9967927486364095, + "pass@64": 0.996992658448461, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9826165594855305, + "pass@2": 0.9903018741929768, + "pass@4": 0.993533330138395, + "pass@8": 0.9951085564879916, + "pass@16": 0.9962574662339553, + "pass@32": 0.9973697274291756, + "pass@64": 0.9983876375026929, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5304828125, + "avg_response_len": 350.74998671875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5304828125, + "pass@2": 0.5999219611220405, + "pass@4": 0.6559194681383493, + "pass@8": 0.7010097915424642, + "pass@16": 0.7381153335387222, + "pass@32": 0.7697573888646824, + "pass@64": 0.7967572463117962, + "pass@128": 0.8201 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9337734375, + "pass@2": 0.9539399606299215, + "pass@4": 0.9656581854143234, + "pass@8": 0.972728971649523, + "pass@16": 0.9772278152236041, + "pass@32": 0.9805309860066493, + "pass@64": 0.9834838573512905, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.889578125, + "pass@2": 0.9227786663385836, + "pass@4": 0.9403892942444698, + "pass@8": 0.9513564248506009, + "pass@16": 0.9590124070467365, + "pass@32": 0.9649496761713952, + "pass@64": 0.9698133678654979, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.7655234375, + "pass@2": 0.8255009842519695, + "pass@4": 0.8629764448818895, + "pass@8": 0.8903779930768185, + "pass@16": 0.9127607899909562, + "pass@32": 0.9312118155747039, + "pass@64": 0.9450888244304412, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.70196875, + "pass@2": 0.780929256889764, + "pass@4": 0.831928569366328, + "pass@8": 0.8661109421572718, + "pass@16": 0.8909653331227422, + "pass@32": 0.910055997504727, + "pass@64": 0.9252640512911471, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.60340625, + "pass@2": 0.6897447096456703, + "pass@4": 0.752191960817398, + "pass@8": 0.798661083343548, + "pass@16": 0.8328480748661413, + "pass@32": 0.8584435273992675, + "pass@64": 0.8780771970348052, + "pass@128": 0.894 + }, + "16": { + "pass@1": 0.417453125, + "pass@2": 0.5178649114173235, + "pass@4": 0.6019887169103861, + "pass@8": 0.668724833540741, + "pass@16": 0.7223904060216403, + "pass@32": 0.7675294126460944, + "pass@64": 0.8058771441760192, + "pass@128": 0.839 + }, + "17": { + "pass@1": 0.3259140625, + "pass@2": 0.41867556594488253, + "pass@4": 0.5003152760592421, + "pass@8": 0.5679369692818151, + "pass@16": 0.625388040595463, + "pass@32": 0.6773521182791972, + "pass@64": 0.7237710055297549, + "pass@128": 0.763 + }, + "18": { + "pass@1": 0.296890625, + "pass@2": 0.38183427657480257, + "pass@4": 0.4592376048931378, + "pass@8": 0.5252002677717833, + "pass@16": 0.5809339021957433, + "pass@32": 0.6295080863140605, + "pass@64": 0.6719924633229738, + "pass@128": 0.71 + }, + "19": { + "pass@1": 0.1954765625, + "pass@2": 0.26490218996063, + "pass@4": 0.3325701462317206, + "pass@8": 0.39420125799474504, + "pass@16": 0.4495748796806342, + "pass@32": 0.5000989095722314, + "pass@64": 0.5452152962782374, + "pass@128": 0.585 + }, + "20": { + "pass@1": 0.17484375, + "pass@2": 0.24304908956692894, + "pass@4": 0.3119384825646792, + "pass@8": 0.3747991717578107, + "pass@16": 0.4300516866434452, + "pass@32": 0.4778933591784907, + "pass@64": 0.5189892558377178, + "pass@128": 0.558 + } + }, + "per_op_accuracy": { + "11": 0.9337734375, + "12": 0.889578125, + "13": 0.7655234375, + "14": 0.70196875, + "15": 0.60340625, + "16": 0.417453125, + "17": 0.3259140625, + "18": 0.296890625, + "19": 0.1954765625, + "20": 0.17484375 + }, + "per_op_avg_response_len": { + "11": 298.62890625, + "12": 309.7895234375, + "13": 327.4443515625, + "14": 349.092390625, + "15": 363.1506953125, + "16": 357.487421875, + "17": 373.812203125, + "18": 378.420609375, + "19": 369.0733359375, + "20": 380.6004296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.62890625, + "12": 309.7895234375, + "13": 327.4443515625, + "14": 349.092390625, + "15": 363.1506953125, + "16": 357.487421875, + "17": 373.812203125, + "18": 378.420609375, + "19": 369.0733359375, + "20": 380.6004296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 229887, + "answer_accuracy": 0.5337272473997028, + "avg_response_len": 332.6431927934621, + "resp_tokens_sum": 143276076, + "pass_at_k": { + "pass@1": 0.5337272473997028, + "pass@2": 0.6012353605316456, + "pass@4": 0.6549324346715222, + "pass@8": 0.6980347716454984, + "pass@16": 0.733964162775554, + "pass@32": 0.765407738415107, + "pass@64": 0.7924486986144456, + "pass@128": 0.8154531946508172 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9278063031161473, + "pass@2": 0.9487857871785149, + "pass@4": 0.9633046656172224, + "pass@8": 0.9732216357284517, + "pass@16": 0.9786658872034295, + "pass@32": 0.9810684775689591, + "pass@64": 0.9829636626890702, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8850635593220338, + "pass@2": 0.9202491770096539, + "pass@4": 0.9391565167171897, + "pass@8": 0.9514797994255895, + "pass@16": 0.9617267819378945, + "pass@32": 0.9705861717353395, + "pass@64": 0.9767466001848372, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7866134404388715, + "pass@2": 0.8468493138005083, + "pass@4": 0.8822010630332648, + "pass@8": 0.9069045661654141, + "pass@16": 0.9283060408067323, + "pass@32": 0.9468951743922206, + "pass@64": 0.9605098936750155, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.7277063862928349, + "pass@2": 0.8057865430372606, + "pass@4": 0.854414851473005, + "pass@8": 0.8872194108437214, + "pass@16": 0.9123645068788976, + "pass@32": 0.9307088495946864, + "pass@64": 0.943620617327961, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.6153134031791907, + "pass@2": 0.6983889985662916, + "pass@4": 0.7580695134387098, + "pass@8": 0.8031574461025102, + "pass@16": 0.837299767040601, + "pass@32": 0.8640886673338127, + "pass@64": 0.8858715415523329, + "pass@128": 0.9046242774566474 + }, + "16": { + "pass@1": 0.4441532258064516, + "pass@2": 0.5437658750317501, + "pass@4": 0.6213555257205754, + "pass@8": 0.6794175538084498, + "pass@16": 0.7247769775563738, + "pass@32": 0.7667628270706329, + "pass@64": 0.8092163638413238, + "pass@128": 0.8516129032258064 + }, + "17": { + "pass@1": 0.3237220014450867, + "pass@2": 0.41972159323653907, + "pass@4": 0.5039649367347865, + "pass@8": 0.572165789931581, + "pass@16": 0.6289312966952066, + "pass@32": 0.6788645306450084, + "pass@64": 0.7199463447879944, + "pass@128": 0.7514450867052023 + }, + "18": { + "pass@1": 0.2625892857142857, + "pass@2": 0.3419112064116983, + "pass@4": 0.41752887326584176, + "pass@8": 0.48695385645562195, + "pass@16": 0.54896983738406, + "pass@32": 0.6050049921493347, + "pass@64": 0.6538989265310818, + "pass@128": 0.6942857142857143 + }, + "19": { + "pass@1": 0.18245589339339338, + "pass@2": 0.25184990600837054, + "pass@4": 0.3204213563957658, + "pass@8": 0.3809279940116799, + "pass@16": 0.4345282615743784, + "pass@32": 0.48636597717050534, + "pass@64": 0.5350908083679881, + "pass@128": 0.5765765765765766 + }, + "20": { + "pass@1": 0.16631475225225226, + "pass@2": 0.22431338819134097, + "pass@4": 0.2808733702769135, + "pass@8": 0.3322876263570989, + "pass@16": 0.37848729652153434, + "pass@32": 0.419235316807995, + "pass@64": 0.4534852920761429, + "pass@128": 0.4864864864864865 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 227683, + "answer_accuracy": 0.5289246022896223, + "avg_response_len": 347.08999358831403, + "resp_tokens_sum": 149409747, + "pass_at_k": { + "pass@1": 0.5289246022896223, + "pass@2": 0.5991446914781305, + "pass@4": 0.6561549001568849, + "pass@8": 0.7027742340371979, + "pass@16": 0.7412912910500087, + "pass@32": 0.7736028603871856, + "pass@64": 0.8001903202339669, + "pass@128": 0.8227772821885222 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9410768072289156, + "pass@2": 0.9577320107200459, + "pass@4": 0.9657074083736515, + "pass@8": 0.9708266595548805, + "pass@16": 0.9748104884529598, + "pass@32": 0.9784491920917664, + "pass@64": 0.9818884198594234, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839145768025078, + "pass@2": 0.9131497914249748, + "pass@4": 0.9279835866911619, + "pass@8": 0.9358928090941657, + "pass@16": 0.9399550162489293, + "pass@32": 0.9430393023055809, + "pass@64": 0.9462353343037858, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.7441132703081232, + "pass@2": 0.8018379595932861, + "pass@4": 0.8393099370981983, + "pass@8": 0.8700972861730256, + "pass@16": 0.8972352305385607, + "pass@32": 0.9193500728527932, + "pass@64": 0.9345754680588337, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6870202850877193, + "pass@2": 0.7691533418520052, + "pass@4": 0.823304849229372, + "pass@8": 0.8601925403880333, + "pass@16": 0.8858854553310859, + "pass@32": 0.904353146068155, + "pass@64": 0.9190286979810856, + "pass@128": 0.9327485380116959 + }, + "15": { + "pass@1": 0.5915719696969697, + "pass@2": 0.6893107999284183, + "pass@4": 0.7606833895763027, + "pass@8": 0.8123372433181013, + "pass@16": 0.8477116751312122, + "pass@32": 0.8719535964129951, + "pass@64": 0.8875869847494624, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4125934829059829, + "pass@2": 0.5122835206496624, + "pass@4": 0.594308329514366, + "pass@8": 0.6581889644775526, + "pass@16": 0.7094556983943399, + "pass@32": 0.7526084755329076, + "pass@64": 0.7885696029137662, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.3438713121118012, + "pass@2": 0.436951325377806, + "pass@4": 0.5197213256922076, + "pass@8": 0.5899766550300463, + "pass@16": 0.6510877878689979, + "pass@32": 0.7069466419261385, + "pass@64": 0.7576633757425218, + "pass@128": 0.8012422360248447 + }, + "18": { + "pass@1": 0.3196920955882353, + "pass@2": 0.4106042293886057, + "pass@4": 0.49403132650698056, + "pass@8": 0.5651694974325554, + "pass@16": 0.6259996375563807, + "pass@32": 0.6794132594830573, + "pass@64": 0.7257263656204008, + "pass@128": 0.7676470588235295 + }, + "19": { + "pass@1": 0.2032412574404762, + "pass@2": 0.2655686105643043, + "pass@4": 0.3245413720606352, + "pass@8": 0.38012602834418274, + "pass@16": 0.43254409374263664, + "pass@32": 0.4813350468599063, + "pass@64": 0.5257167589625654, + "pass@128": 0.5714285714285714 + }, + "20": { + "pass@1": 0.1676880613772455, + "pass@2": 0.23813261763873822, + "pass@4": 0.31379891735090004, + "pass@8": 0.3856492144322349, + "pass@16": 0.44792010040615005, + "pass@32": 0.49754141889425796, + "pass@64": 0.5335500754567829, + "pass@128": 0.562874251497006 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 221448, + "answer_accuracy": 0.5287477078239609, + "avg_response_len": 373.13321363080684, + "resp_tokens_sum": 156274160, + "pass_at_k": { + "pass@1": 0.5287477078239609, + "pass@2": 0.5993701178455242, + "pass@4": 0.6566925762488756, + "pass@8": 0.7022558556755771, + "pass@16": 0.7391202065544705, + "pass@32": 0.7702782485934773, + "pass@64": 0.797659702119013, + "pass@128": 0.8221271393643031 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9327628968253968, + "pass@2": 0.9557192069741284, + "pass@4": 0.9682437424488604, + "pass@8": 0.9741818516354278, + "pass@16": 0.9781640472203527, + "pass@32": 0.9821227988899696, + "pass@64": 0.9857483461229213, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8999904434250765, + "pass@2": 0.9349103186929618, + "pass@4": 0.9538260647464932, + "pass@8": 0.9663081643819669, + "pass@16": 0.9746650643954538, + "pass@32": 0.980222122145579, + "pass@64": 0.9853088677588914, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7683497299382716, + "pass@2": 0.8305552517740837, + "pass@4": 0.8701254883648802, + "pass@8": 0.896452794155191, + "pass@16": 0.9145623014547616, + "pass@32": 0.9288403670837605, + "pass@64": 0.9414898958367518, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6926233308605341, + "pass@2": 0.7692027705086564, + "pass@4": 0.8192615537301153, + "pass@8": 0.8520108678448942, + "pass@16": 0.8757373907756257, + "pass@32": 0.8961711597315235, + "pass@64": 0.9141068795826112, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.6027440200617284, + "pass@2": 0.6809554079177604, + "pass@4": 0.7372666376772344, + "pass@8": 0.779929989805882, + "pass@16": 0.8129552551752913, + "pass@32": 0.838654820016913, + "pass@64": 0.8600677120073306, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.3980687684365782, + "pass@2": 0.49995862657654505, + "pass@4": 0.5922309152727587, + "pass@8": 0.66985564993658, + "pass@16": 0.733600568857674, + "pass@32": 0.7836795319827353, + "pass@64": 0.8207437780603994, + "pass@128": 0.8525073746312685 + }, + "17": { + "pass@1": 0.31078219126506024, + "pass@2": 0.3998601443174271, + "pass@4": 0.4776901839642033, + "pass@8": 0.5421539850777513, + "pass@16": 0.5967697115816433, + "pass@32": 0.6470728071560454, + "pass@64": 0.6948854314578804, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.31060987903225806, + "pass@2": 0.3953545688341375, + "pass@4": 0.46816725237974255, + "pass@8": 0.5245444802749871, + "pass@16": 0.5675954269101743, + "pass@32": 0.6024381639921098, + "pass@64": 0.6334866926650284, + "pass@128": 0.6645161290322581 + }, + "19": { + "pass@1": 0.20069391993957703, + "pass@2": 0.27735685229678597, + "pass@4": 0.352942397400476, + "pass@8": 0.4218425693813008, + "pass@16": 0.48200046247383743, + "pass@32": 0.5329621251647273, + "pass@64": 0.5751940364962989, + "pass@128": 0.6072507552870091 + }, + "20": { + "pass@1": 0.19054992492492492, + "pass@2": 0.26671602705067277, + "pass@4": 0.3411375731749749, + "pass@8": 0.40642809177336414, + "pass@16": 0.4636940041024634, + "pass@32": 0.5168443386508894, + "pass@64": 0.5698886737951843, + "pass@128": 0.6246246246246246 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7464560032894737, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7464560032894737, + "pass@2": 0.7848114704206178, + "pass@4": 0.8153904222662789, + "pass@8": 0.8399657722909037, + "pass@16": 0.8601820744773508, + "pass@32": 0.8773836407181179, + "pass@64": 0.8920095742633558, + "pass@128": 0.904578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.94303125, + "pass@2": 0.9573801673228353, + "pass@4": 0.9675857014435695, + "pass@8": 0.9757808435253364, + "pass@16": 0.9823870776171006, + "pass@32": 0.9876964807576623, + "pass@64": 0.9919439701585595, + "pass@128": 0.995 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999765625, + "pass@2": 0.9999808070866142, + "pass@4": 0.9999997759655043, + "pass@8": 0.9999999999829964, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9970859375, + "pass@2": 0.9977909694881889, + "pass@4": 0.9981468881702288, + "pass@8": 0.9983682961001835, + "pass@16": 0.9986167377672557, + "pass@32": 0.9988739053984702, + "pass@64": 0.9989934280775878, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.984890625, + "pass@2": 0.9885019685039373, + "pass@4": 0.9909092400637421, + "pass@8": 0.9931073260534368, + "pass@16": 0.9949935107762257, + "pass@32": 0.9963786095513063, + "pass@64": 0.9974462093451094, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9942109375, + "pass@2": 0.9950751722440945, + "pass@4": 0.9955810483689539, + "pass@8": 0.9960776518199143, + "pass@16": 0.9966914915112858, + "pass@32": 0.9972938163882501, + "pass@64": 0.9977453345301317, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9901796875, + "pass@2": 0.9934657972440946, + "pass@4": 0.9952312505624299, + "pass@8": 0.9963188584246357, + "pass@16": 0.9972675226811877, + "pass@32": 0.9981704078751042, + "pass@64": 0.9987416329818158, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.985828125, + "pass@2": 0.990656126968504, + "pass@4": 0.9932887762467191, + "pass@8": 0.9950519033029102, + "pass@16": 0.9964262465725706, + "pass@32": 0.9975526463198227, + "pass@64": 0.9983623777967937, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9828515625, + "pass@2": 0.989347317913386, + "pass@4": 0.9924806608548928, + "pass@8": 0.9945468788929211, + "pass@16": 0.9959234927567568, + "pass@32": 0.9967494187065778, + "pass@64": 0.9973764949958263, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.9337734375, + "pass@2": 0.9539399606299215, + "pass@4": 0.9656581854143234, + "pass@8": 0.972728971649523, + "pass@16": 0.9772278152236041, + "pass@32": 0.9805309860066493, + "pass@64": 0.9834838573512905, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.889578125, + "pass@2": 0.9227786663385836, + "pass@4": 0.9403892942444698, + "pass@8": 0.9513564248506009, + "pass@16": 0.9590124070467365, + "pass@32": 0.9649496761713952, + "pass@64": 0.9698133678654979, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.7655234375, + "pass@2": 0.8255009842519695, + "pass@4": 0.8629764448818895, + "pass@8": 0.8903779930768185, + "pass@16": 0.9127607899909562, + "pass@32": 0.9312118155747039, + "pass@64": 0.9450888244304412, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.70196875, + "pass@2": 0.780929256889764, + "pass@4": 0.831928569366328, + "pass@8": 0.8661109421572718, + "pass@16": 0.8909653331227422, + "pass@32": 0.910055997504727, + "pass@64": 0.9252640512911471, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.60340625, + "pass@2": 0.6897447096456703, + "pass@4": 0.752191960817398, + "pass@8": 0.798661083343548, + "pass@16": 0.8328480748661413, + "pass@32": 0.8584435273992675, + "pass@64": 0.8780771970348052, + "pass@128": 0.894 + }, + "16": { + "pass@1": 0.417453125, + "pass@2": 0.5178649114173235, + "pass@4": 0.6019887169103861, + "pass@8": 0.668724833540741, + "pass@16": 0.7223904060216403, + "pass@32": 0.7675294126460944, + "pass@64": 0.8058771441760192, + "pass@128": 0.839 + }, + "17": { + "pass@1": 0.3259140625, + "pass@2": 0.41867556594488253, + "pass@4": 0.5003152760592421, + "pass@8": 0.5679369692818151, + "pass@16": 0.625388040595463, + "pass@32": 0.6773521182791972, + "pass@64": 0.7237710055297549, + "pass@128": 0.763 + }, + "18": { + "pass@1": 0.296890625, + "pass@2": 0.38183427657480257, + "pass@4": 0.4592376048931378, + "pass@8": 0.5252002677717833, + "pass@16": 0.5809339021957433, + "pass@32": 0.6295080863140605, + "pass@64": 0.6719924633229738, + "pass@128": 0.71 + }, + "19": { + "pass@1": 0.1954765625, + "pass@2": 0.26490218996063, + "pass@4": 0.3325701462317206, + "pass@8": 0.39420125799474504, + "pass@16": 0.4495748796806342, + "pass@32": 0.5000989095722314, + "pass@64": 0.5452152962782374, + "pass@128": 0.585 + }, + "20": { + "pass@1": 0.17484375, + "pass@2": 0.24304908956692894, + "pass@4": 0.3119384825646792, + "pass@8": 0.3747991717578107, + "pass@16": 0.4300516866434452, + "pass@32": 0.4778933591784907, + "pass@64": 0.5189892558377178, + "pass@128": 0.558 + } + }, + "per_op_accuracy": { + "10": 0.94303125, + "2": 0.9999921875, + "3": 0.999765625, + "4": 0.9970859375, + "5": 0.984890625, + "6": 0.9942109375, + "7": 0.9901796875, + "8": 0.985828125, + "9": 0.9828515625, + "11": 0.9337734375, + "12": 0.889578125, + "13": 0.7655234375, + "14": 0.70196875, + "15": 0.60340625, + "16": 0.417453125, + "17": 0.3259140625, + "18": 0.296890625, + "19": 0.1954765625, + "20": 0.17484375 + }, + "per_op_avg_response_len": { + "10": 280.6092265625, + "2": 115.6311328125, + "3": 144.821484375, + "4": 160.1008984375, + "5": 174.0573125, + "6": 199.284875, + "7": 224.164765625, + "8": 243.8772109375, + "9": 259.5847734375, + "11": 298.62890625, + "12": 309.7895234375, + "13": 327.4443515625, + "14": 349.092390625, + "15": 363.1506953125, + "16": 357.487421875, + "17": 373.812203125, + "18": 378.420609375, + "19": 369.0733359375, + "20": 380.6004296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.6092265625, + "2": 115.6311328125, + "3": 144.821484375, + "4": 160.1008984375, + "5": 174.0573125, + "6": 199.284875, + "7": 224.164765625, + "8": 243.8772109375, + "9": 259.5847734375, + "11": 298.62890625, + "12": 309.7895234375, + "13": 327.4443515625, + "14": 349.092390625, + "15": 363.1506953125, + "16": 357.487421875, + "17": 373.812203125, + "18": 378.420609375, + "19": 369.0733359375, + "20": 380.6004296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 603531, + "answer_accuracy": 0.7453502904679102, + "avg_response_len": 266.2402337575087, + "resp_tokens_sum": 215582172, + "pass_at_k": { + "pass@1": 0.7453502904679102, + "pass@2": 0.7831601583018162, + "pass@4": 0.8129134922047506, + "pass@8": 0.8366994810292484, + "pass@16": 0.8564386661569456, + "pass@32": 0.8737103539945428, + "pass@64": 0.8885903356765203, + "pass@128": 0.9012013910844136 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412103613569321, + "pass@2": 0.9582161086335446, + "pass@4": 0.9704242616796793, + "pass@8": 0.978816878970928, + "pass@16": 0.9845971539341197, + "pass@32": 0.9898851916877327, + "pass@64": 0.9949850127164509, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998566513761468, + "pass@2": 0.9999962375689279, + "pass@4": 0.999999998566693, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.9940810316190944, + "pass@4": 0.9943864211309524, + "pass@8": 0.9949083553134181, + "pass@16": 0.9956773146738891, + "pass@32": 0.9964809543702204, + "pass@64": 0.996854462742462, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.985218253968254, + "pass@2": 0.9877667635295586, + "pass@4": 0.9895235125371236, + "pass@8": 0.990932228795399, + "pass@16": 0.9919952093986906, + "pass@32": 0.992917042399551, + "pass@64": 0.9935593050866022, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9928892215568862, + "pass@2": 0.9938966800650667, + "pass@4": 0.9941838760611511, + "pass@8": 0.9943758755054594, + "pass@16": 0.994716276109128, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9916327662721893, + "pass@2": 0.9952126916088152, + "pass@4": 0.9968094700070774, + "pass@8": 0.9973776120848545, + "pass@16": 0.9977373270741638, + "pass@32": 0.9983401667986244, + "pass@64": 0.9992661790057308, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9821464342948718, + "pass@2": 0.9871637139107611, + "pass@4": 0.9906239857037101, + "pass@8": 0.9937512123492258, + "pass@16": 0.9961718325495631, + "pass@32": 0.998099042572955, + "pass@64": 0.9995645131644167, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9873239436619718, + "pass@2": 0.9934907674392813, + "pass@4": 0.9963719001674085, + "pass@8": 0.9984650960718653, + "pass@16": 0.999695388738615, + "pass@32": 0.9999885966972071, + "pass@64": 0.9999999973129757, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9278063031161473, + "pass@2": 0.9487857871785149, + "pass@4": 0.9633046656172224, + "pass@8": 0.9732216357284517, + "pass@16": 0.9786658872034295, + "pass@32": 0.9810684775689591, + "pass@64": 0.9829636626890702, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8850635593220338, + "pass@2": 0.9202491770096539, + "pass@4": 0.9391565167171897, + "pass@8": 0.9514797994255895, + "pass@16": 0.9617267819378945, + "pass@32": 0.9705861717353395, + "pass@64": 0.9767466001848372, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7866134404388715, + "pass@2": 0.8468493138005083, + "pass@4": 0.8822010630332648, + "pass@8": 0.9069045661654141, + "pass@16": 0.9283060408067323, + "pass@32": 0.9468951743922206, + "pass@64": 0.9605098936750155, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.7277063862928349, + "pass@2": 0.8057865430372606, + "pass@4": 0.854414851473005, + "pass@8": 0.8872194108437214, + "pass@16": 0.9123645068788976, + "pass@32": 0.9307088495946864, + "pass@64": 0.943620617327961, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.6153134031791907, + "pass@2": 0.6983889985662916, + "pass@4": 0.7580695134387098, + "pass@8": 0.8031574461025102, + "pass@16": 0.837299767040601, + "pass@32": 0.8640886673338127, + "pass@64": 0.8858715415523329, + "pass@128": 0.9046242774566474 + }, + "16": { + "pass@1": 0.4441532258064516, + "pass@2": 0.5437658750317501, + "pass@4": 0.6213555257205754, + "pass@8": 0.6794175538084498, + "pass@16": 0.7247769775563738, + "pass@32": 0.7667628270706329, + "pass@64": 0.8092163638413238, + "pass@128": 0.8516129032258064 + }, + "17": { + "pass@1": 0.3237220014450867, + "pass@2": 0.41972159323653907, + "pass@4": 0.5039649367347865, + "pass@8": 0.572165789931581, + "pass@16": 0.6289312966952066, + "pass@32": 0.6788645306450084, + "pass@64": 0.7199463447879944, + "pass@128": 0.7514450867052023 + }, + "18": { + "pass@1": 0.2625892857142857, + "pass@2": 0.3419112064116983, + "pass@4": 0.41752887326584176, + "pass@8": 0.48695385645562195, + "pass@16": 0.54896983738406, + "pass@32": 0.6050049921493347, + "pass@64": 0.6538989265310818, + "pass@128": 0.6942857142857143 + }, + "19": { + "pass@1": 0.18245589339339338, + "pass@2": 0.25184990600837054, + "pass@4": 0.3204213563957658, + "pass@8": 0.3809279940116799, + "pass@16": 0.4345282615743784, + "pass@32": 0.48636597717050534, + "pass@64": 0.5350908083679881, + "pass@128": 0.5765765765765766 + }, + "20": { + "pass@1": 0.16631475225225226, + "pass@2": 0.22431338819134097, + "pass@4": 0.2808733702769135, + "pass@8": 0.3322876263570989, + "pass@16": 0.37848729652153434, + "pass@32": 0.419235316807995, + "pass@64": 0.4534852920761429, + "pass@128": 0.4864864864864865 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 613318, + "answer_accuracy": 0.7469285853468434, + "avg_response_len": 276.28852786438034, + "resp_tokens_sum": 226866036, + "pass_at_k": { + "pass@1": 0.7469285853468434, + "pass@2": 0.7852395192124672, + "pass@4": 0.8160709843995477, + "pass@8": 0.8413264985156623, + "pass@16": 0.8622231992632267, + "pass@32": 0.8796793463115843, + "pass@64": 0.8940219495111036, + "pass@128": 0.906157443491816 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9334499616564417, + "pass@2": 0.9457872657118013, + "pass@4": 0.9571576134463252, + "pass@8": 0.9686398110946103, + "pass@16": 0.9785876056618497, + "pass@32": 0.9856178979361547, + "pass@64": 0.9907399312937869, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998579545454546, + "pass@2": 0.9999962717728466, + "pass@4": 0.9999999985797231, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981971153846154, + "pass@2": 0.999273458975912, + "pass@4": 0.9998449130619028, + "pass@8": 0.9999930032520015, + "pass@16": 0.9999999913361977, + "pass@32": 0.999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991304347826087, + "pass@2": 0.9943758558712769, + "pass@4": 0.9953860142482189, + "pass@8": 0.9962545915555301, + "pass@16": 0.9973181192300502, + "pass@32": 0.998318582579071, + "pass@64": 0.999280888268243, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968660458452722, + "pass@2": 0.9971247997653588, + "pass@4": 0.9971346516857312, + "pass@8": 0.997134670487104, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9948292991329479, + "pass@2": 0.996495024691639, + "pass@4": 0.9970465990342248, + "pass@8": 0.9971089117505242, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9894586267605634, + "pass@2": 0.9937756460019963, + "pass@4": 0.9952096093622099, + "pass@8": 0.9960140820363009, + "pass@16": 0.9969277354949511, + "pass@32": 0.9977852387074259, + "pass@64": 0.9985906885243891, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9783168038922155, + "pass@2": 0.9840545287378, + "pass@4": 0.9873645827018628, + "pass@8": 0.9898593075438438, + "pass@16": 0.9916034663346961, + "pass@32": 0.9927289869718435, + "pass@64": 0.993646529001145, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9410768072289156, + "pass@2": 0.9577320107200459, + "pass@4": 0.9657074083736515, + "pass@8": 0.9708266595548805, + "pass@16": 0.9748104884529598, + "pass@32": 0.9784491920917664, + "pass@64": 0.9818884198594234, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839145768025078, + "pass@2": 0.9131497914249748, + "pass@4": 0.9279835866911619, + "pass@8": 0.9358928090941657, + "pass@16": 0.9399550162489293, + "pass@32": 0.9430393023055809, + "pass@64": 0.9462353343037858, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.7441132703081232, + "pass@2": 0.8018379595932861, + "pass@4": 0.8393099370981983, + "pass@8": 0.8700972861730256, + "pass@16": 0.8972352305385607, + "pass@32": 0.9193500728527932, + "pass@64": 0.9345754680588337, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6870202850877193, + "pass@2": 0.7691533418520052, + "pass@4": 0.823304849229372, + "pass@8": 0.8601925403880333, + "pass@16": 0.8858854553310859, + "pass@32": 0.904353146068155, + "pass@64": 0.9190286979810856, + "pass@128": 0.9327485380116959 + }, + "15": { + "pass@1": 0.5915719696969697, + "pass@2": 0.6893107999284183, + "pass@4": 0.7606833895763027, + "pass@8": 0.8123372433181013, + "pass@16": 0.8477116751312122, + "pass@32": 0.8719535964129951, + "pass@64": 0.8875869847494624, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4125934829059829, + "pass@2": 0.5122835206496624, + "pass@4": 0.594308329514366, + "pass@8": 0.6581889644775526, + "pass@16": 0.7094556983943399, + "pass@32": 0.7526084755329076, + "pass@64": 0.7885696029137662, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.3438713121118012, + "pass@2": 0.436951325377806, + "pass@4": 0.5197213256922076, + "pass@8": 0.5899766550300463, + "pass@16": 0.6510877878689979, + "pass@32": 0.7069466419261385, + "pass@64": 0.7576633757425218, + "pass@128": 0.8012422360248447 + }, + "18": { + "pass@1": 0.3196920955882353, + "pass@2": 0.4106042293886057, + "pass@4": 0.49403132650698056, + "pass@8": 0.5651694974325554, + "pass@16": 0.6259996375563807, + "pass@32": 0.6794132594830573, + "pass@64": 0.7257263656204008, + "pass@128": 0.7676470588235295 + }, + "19": { + "pass@1": 0.2032412574404762, + "pass@2": 0.2655686105643043, + "pass@4": 0.3245413720606352, + "pass@8": 0.38012602834418274, + "pass@16": 0.43254409374263664, + "pass@32": 0.4813350468599063, + "pass@64": 0.5257167589625654, + "pass@128": 0.5714285714285714 + }, + "20": { + "pass@1": 0.1676880613772455, + "pass@2": 0.23813261763873822, + "pass@4": 0.31379891735090004, + "pass@8": 0.3856492144322349, + "pass@16": 0.44792010040615005, + "pass@32": 0.49754141889425796, + "pass@64": 0.5335500754567829, + "pass@128": 0.562874251497006 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 598532, + "answer_accuracy": 0.7470891915641477, + "avg_response_len": 296.05446906454705, + "resp_tokens_sum": 237184630, + "pass_at_k": { + "pass@1": 0.7470891915641477, + "pass@2": 0.786041741624341, + "pass@4": 0.8171963422989587, + "pass@8": 0.8418723867323799, + "pass@16": 0.8618735564286889, + "pass@32": 0.8787433244425196, + "pass@64": 0.8934028824733017, + "pass@128": 0.9063748202588273 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9541977611940299, + "pass@2": 0.9678156951463157, + "pass@4": 0.9748611783601676, + "pass@8": 0.9796577407085042, + "pass@16": 0.9838480089184216, + "pass@32": 0.9875043762636851, + "pass@64": 0.9900383081967405, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999512178324649, + "pass@4": 0.9999993495710995, + "pass@8": 0.9999999999504271, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999109100877193, + "pass@2": 0.9997971059538611, + "pass@4": 0.9999872888695931, + "pass@8": 0.9999999628675824, + "pass@16": 0.999999999999931, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.978079044117647, + "pass@2": 0.9832228462251044, + "pass@4": 0.9876504667615077, + "pass@8": 0.9919289408712592, + "pass@16": 0.9954125549449143, + "pass@32": 0.9976171476637306, + "pass@64": 0.9991856523243703, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9926804022082019, + "pass@2": 0.9940603343351797, + "pass@4": 0.9953427139628053, + "pass@8": 0.9967069697195312, + "pass@16": 0.998284717005795, + "pass@32": 0.9995420820514109, + "pass@64": 0.9999790726378381, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.983534414556962, + "pass@2": 0.9882804713196452, + "pass@4": 0.9915554633993535, + "pass@8": 0.9943213356780815, + "pass@16": 0.9969376777404727, + "pass@32": 0.9991500996745862, + "pass@64": 0.9999672926515153, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9854072822822822, + "pass@2": 0.9906026843772905, + "pass@4": 0.9937377819326637, + "pass@8": 0.9952448226037982, + "pass@16": 0.9961299961453437, + "pass@32": 0.9967927486364095, + "pass@64": 0.996992658448461, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9826165594855305, + "pass@2": 0.9903018741929768, + "pass@4": 0.993533330138395, + "pass@8": 0.9951085564879916, + "pass@16": 0.9962574662339553, + "pass@32": 0.9973697274291756, + "pass@64": 0.9983876375026929, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9327628968253968, + "pass@2": 0.9557192069741284, + "pass@4": 0.9682437424488604, + "pass@8": 0.9741818516354278, + "pass@16": 0.9781640472203527, + "pass@32": 0.9821227988899696, + "pass@64": 0.9857483461229213, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8999904434250765, + "pass@2": 0.9349103186929618, + "pass@4": 0.9538260647464932, + "pass@8": 0.9663081643819669, + "pass@16": 0.9746650643954538, + "pass@32": 0.980222122145579, + "pass@64": 0.9853088677588914, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7683497299382716, + "pass@2": 0.8305552517740837, + "pass@4": 0.8701254883648802, + "pass@8": 0.896452794155191, + "pass@16": 0.9145623014547616, + "pass@32": 0.9288403670837605, + "pass@64": 0.9414898958367518, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6926233308605341, + "pass@2": 0.7692027705086564, + "pass@4": 0.8192615537301153, + "pass@8": 0.8520108678448942, + "pass@16": 0.8757373907756257, + "pass@32": 0.8961711597315235, + "pass@64": 0.9141068795826112, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.6027440200617284, + "pass@2": 0.6809554079177604, + "pass@4": 0.7372666376772344, + "pass@8": 0.779929989805882, + "pass@16": 0.8129552551752913, + "pass@32": 0.838654820016913, + "pass@64": 0.8600677120073306, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.3980687684365782, + "pass@2": 0.49995862657654505, + "pass@4": 0.5922309152727587, + "pass@8": 0.66985564993658, + "pass@16": 0.733600568857674, + "pass@32": 0.7836795319827353, + "pass@64": 0.8207437780603994, + "pass@128": 0.8525073746312685 + }, + "17": { + "pass@1": 0.31078219126506024, + "pass@2": 0.3998601443174271, + "pass@4": 0.4776901839642033, + "pass@8": 0.5421539850777513, + "pass@16": 0.5967697115816433, + "pass@32": 0.6470728071560454, + "pass@64": 0.6948854314578804, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.31060987903225806, + "pass@2": 0.3953545688341375, + "pass@4": 0.46816725237974255, + "pass@8": 0.5245444802749871, + "pass@16": 0.5675954269101743, + "pass@32": 0.6024381639921098, + "pass@64": 0.6334866926650284, + "pass@128": 0.6645161290322581 + }, + "19": { + "pass@1": 0.20069391993957703, + "pass@2": 0.27735685229678597, + "pass@4": 0.352942397400476, + "pass@8": 0.4218425693813008, + "pass@16": 0.48200046247383743, + "pass@32": 0.5329621251647273, + "pass@64": 0.5751940364962989, + "pass@128": 0.6072507552870091 + }, + "20": { + "pass@1": 0.19054992492492492, + "pass@2": 0.26671602705067277, + "pass@4": 0.3411375731749749, + "pass@8": 0.40642809177336414, + "pass@16": 0.4636940041024634, + "pass@32": 0.5168443386508894, + "pass@64": 0.5698886737951843, + "pass@128": 0.6246246246246246 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b7c79ab27eded3ac3b839870abedc192afe94e84 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1d4fc80ca14230d72362509170c2b6bb050c2fe6abb97a47c94f874a0c75230 +size 2635457456 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/summary.csv b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..fa1809c7629d5e4205746ac0a490e020ff840341 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op9-12,0.9864262152777777,0.5304828125,0.7464560032894737,0.0,0.0,0.0,200.23685329861112,350.74998671875,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/summary.json b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1b60fece741a1e5814bb938b2510c8af32ce665c --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op9-12", + "id_acc": 0.9864262152777777, + "ood_acc": 0.5304828125, + "total_acc": 0.7464560032894737, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.23685329861112, + "ood_avg_resp_len": 350.74998671875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 603531, + "answer_accuracy": 0.7453502904679102, + "avg_response_len": 266.2402337575087, + "resp_tokens_sum": 215582172, + "pass_at_k": { + "pass@1": 0.7453502904679102, + "pass@2": 0.7831601583018162, + "pass@4": 0.8129134922047506, + "pass@8": 0.8366994810292484, + "pass@16": 0.8564386661569456, + "pass@32": 0.8737103539945428, + "pass@64": 0.8885903356765203, + "pass@128": 0.9012013910844136 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412103613569321, + "pass@2": 0.9582161086335446, + "pass@4": 0.9704242616796793, + "pass@8": 0.978816878970928, + "pass@16": 0.9845971539341197, + "pass@32": 0.9898851916877327, + "pass@64": 0.9949850127164509, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998566513761468, + "pass@2": 0.9999962375689279, + "pass@4": 0.999999998566693, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.9940810316190944, + "pass@4": 0.9943864211309524, + "pass@8": 0.9949083553134181, + "pass@16": 0.9956773146738891, + "pass@32": 0.9964809543702204, + "pass@64": 0.996854462742462, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.985218253968254, + "pass@2": 0.9877667635295586, + "pass@4": 0.9895235125371236, + "pass@8": 0.990932228795399, + "pass@16": 0.9919952093986906, + "pass@32": 0.992917042399551, + "pass@64": 0.9935593050866022, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9928892215568862, + "pass@2": 0.9938966800650667, + "pass@4": 0.9941838760611511, + "pass@8": 0.9943758755054594, + "pass@16": 0.994716276109128, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9916327662721893, + "pass@2": 0.9952126916088152, + "pass@4": 0.9968094700070774, + "pass@8": 0.9973776120848545, + "pass@16": 0.9977373270741638, + "pass@32": 0.9983401667986244, + "pass@64": 0.9992661790057308, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9821464342948718, + "pass@2": 0.9871637139107611, + "pass@4": 0.9906239857037101, + "pass@8": 0.9937512123492258, + "pass@16": 0.9961718325495631, + "pass@32": 0.998099042572955, + "pass@64": 0.9995645131644167, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9873239436619718, + "pass@2": 0.9934907674392813, + "pass@4": 0.9963719001674085, + "pass@8": 0.9984650960718653, + "pass@16": 0.999695388738615, + "pass@32": 0.9999885966972071, + "pass@64": 0.9999999973129757, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9278063031161473, + "pass@2": 0.9487857871785149, + "pass@4": 0.9633046656172224, + "pass@8": 0.9732216357284517, + "pass@16": 0.9786658872034295, + "pass@32": 0.9810684775689591, + "pass@64": 0.9829636626890702, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8850635593220338, + "pass@2": 0.9202491770096539, + "pass@4": 0.9391565167171897, + "pass@8": 0.9514797994255895, + "pass@16": 0.9617267819378945, + "pass@32": 0.9705861717353395, + "pass@64": 0.9767466001848372, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7866134404388715, + "pass@2": 0.8468493138005083, + "pass@4": 0.8822010630332648, + "pass@8": 0.9069045661654141, + "pass@16": 0.9283060408067323, + "pass@32": 0.9468951743922206, + "pass@64": 0.9605098936750155, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.7277063862928349, + "pass@2": 0.8057865430372606, + "pass@4": 0.854414851473005, + "pass@8": 0.8872194108437214, + "pass@16": 0.9123645068788976, + "pass@32": 0.9307088495946864, + "pass@64": 0.943620617327961, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.6153134031791907, + "pass@2": 0.6983889985662916, + "pass@4": 0.7580695134387098, + "pass@8": 0.8031574461025102, + "pass@16": 0.837299767040601, + "pass@32": 0.8640886673338127, + "pass@64": 0.8858715415523329, + "pass@128": 0.9046242774566474 + }, + "16": { + "pass@1": 0.4441532258064516, + "pass@2": 0.5437658750317501, + "pass@4": 0.6213555257205754, + "pass@8": 0.6794175538084498, + "pass@16": 0.7247769775563738, + "pass@32": 0.7667628270706329, + "pass@64": 0.8092163638413238, + "pass@128": 0.8516129032258064 + }, + "17": { + "pass@1": 0.3237220014450867, + "pass@2": 0.41972159323653907, + "pass@4": 0.5039649367347865, + "pass@8": 0.572165789931581, + "pass@16": 0.6289312966952066, + "pass@32": 0.6788645306450084, + "pass@64": 0.7199463447879944, + "pass@128": 0.7514450867052023 + }, + "18": { + "pass@1": 0.2625892857142857, + "pass@2": 0.3419112064116983, + "pass@4": 0.41752887326584176, + "pass@8": 0.48695385645562195, + "pass@16": 0.54896983738406, + "pass@32": 0.6050049921493347, + "pass@64": 0.6538989265310818, + "pass@128": 0.6942857142857143 + }, + "19": { + "pass@1": 0.18245589339339338, + "pass@2": 0.25184990600837054, + "pass@4": 0.3204213563957658, + "pass@8": 0.3809279940116799, + "pass@16": 0.4345282615743784, + "pass@32": 0.48636597717050534, + "pass@64": 0.5350908083679881, + "pass@128": 0.5765765765765766 + }, + "20": { + "pass@1": 0.16631475225225226, + "pass@2": 0.22431338819134097, + "pass@4": 0.2808733702769135, + "pass@8": 0.3322876263570989, + "pass@16": 0.37848729652153434, + "pass@32": 0.419235316807995, + "pass@64": 0.4534852920761429, + "pass@128": 0.4864864864864865 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 613318, + "answer_accuracy": 0.7469285853468434, + "avg_response_len": 276.28852786438034, + "resp_tokens_sum": 226866036, + "pass_at_k": { + "pass@1": 0.7469285853468434, + "pass@2": 0.7852395192124672, + "pass@4": 0.8160709843995477, + "pass@8": 0.8413264985156623, + "pass@16": 0.8622231992632267, + "pass@32": 0.8796793463115843, + "pass@64": 0.8940219495111036, + "pass@128": 0.906157443491816 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9334499616564417, + "pass@2": 0.9457872657118013, + "pass@4": 0.9571576134463252, + "pass@8": 0.9686398110946103, + "pass@16": 0.9785876056618497, + "pass@32": 0.9856178979361547, + "pass@64": 0.9907399312937869, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998579545454546, + "pass@2": 0.9999962717728466, + "pass@4": 0.9999999985797231, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981971153846154, + "pass@2": 0.999273458975912, + "pass@4": 0.9998449130619028, + "pass@8": 0.9999930032520015, + "pass@16": 0.9999999913361977, + "pass@32": 0.999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991304347826087, + "pass@2": 0.9943758558712769, + "pass@4": 0.9953860142482189, + "pass@8": 0.9962545915555301, + "pass@16": 0.9973181192300502, + "pass@32": 0.998318582579071, + "pass@64": 0.999280888268243, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968660458452722, + "pass@2": 0.9971247997653588, + "pass@4": 0.9971346516857312, + "pass@8": 0.997134670487104, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9948292991329479, + "pass@2": 0.996495024691639, + "pass@4": 0.9970465990342248, + "pass@8": 0.9971089117505242, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9894586267605634, + "pass@2": 0.9937756460019963, + "pass@4": 0.9952096093622099, + "pass@8": 0.9960140820363009, + "pass@16": 0.9969277354949511, + "pass@32": 0.9977852387074259, + "pass@64": 0.9985906885243891, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9783168038922155, + "pass@2": 0.9840545287378, + "pass@4": 0.9873645827018628, + "pass@8": 0.9898593075438438, + "pass@16": 0.9916034663346961, + "pass@32": 0.9927289869718435, + "pass@64": 0.993646529001145, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9410768072289156, + "pass@2": 0.9577320107200459, + "pass@4": 0.9657074083736515, + "pass@8": 0.9708266595548805, + "pass@16": 0.9748104884529598, + "pass@32": 0.9784491920917664, + "pass@64": 0.9818884198594234, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839145768025078, + "pass@2": 0.9131497914249748, + "pass@4": 0.9279835866911619, + "pass@8": 0.9358928090941657, + "pass@16": 0.9399550162489293, + "pass@32": 0.9430393023055809, + "pass@64": 0.9462353343037858, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.7441132703081232, + "pass@2": 0.8018379595932861, + "pass@4": 0.8393099370981983, + "pass@8": 0.8700972861730256, + "pass@16": 0.8972352305385607, + "pass@32": 0.9193500728527932, + "pass@64": 0.9345754680588337, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6870202850877193, + "pass@2": 0.7691533418520052, + "pass@4": 0.823304849229372, + "pass@8": 0.8601925403880333, + "pass@16": 0.8858854553310859, + "pass@32": 0.904353146068155, + "pass@64": 0.9190286979810856, + "pass@128": 0.9327485380116959 + }, + "15": { + "pass@1": 0.5915719696969697, + "pass@2": 0.6893107999284183, + "pass@4": 0.7606833895763027, + "pass@8": 0.8123372433181013, + "pass@16": 0.8477116751312122, + "pass@32": 0.8719535964129951, + "pass@64": 0.8875869847494624, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4125934829059829, + "pass@2": 0.5122835206496624, + "pass@4": 0.594308329514366, + "pass@8": 0.6581889644775526, + "pass@16": 0.7094556983943399, + "pass@32": 0.7526084755329076, + "pass@64": 0.7885696029137662, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.3438713121118012, + "pass@2": 0.436951325377806, + "pass@4": 0.5197213256922076, + "pass@8": 0.5899766550300463, + "pass@16": 0.6510877878689979, + "pass@32": 0.7069466419261385, + "pass@64": 0.7576633757425218, + "pass@128": 0.8012422360248447 + }, + "18": { + "pass@1": 0.3196920955882353, + "pass@2": 0.4106042293886057, + "pass@4": 0.49403132650698056, + "pass@8": 0.5651694974325554, + "pass@16": 0.6259996375563807, + "pass@32": 0.6794132594830573, + "pass@64": 0.7257263656204008, + "pass@128": 0.7676470588235295 + }, + "19": { + "pass@1": 0.2032412574404762, + "pass@2": 0.2655686105643043, + "pass@4": 0.3245413720606352, + "pass@8": 0.38012602834418274, + "pass@16": 0.43254409374263664, + "pass@32": 0.4813350468599063, + "pass@64": 0.5257167589625654, + "pass@128": 0.5714285714285714 + }, + "20": { + "pass@1": 0.1676880613772455, + "pass@2": 0.23813261763873822, + "pass@4": 0.31379891735090004, + "pass@8": 0.3856492144322349, + "pass@16": 0.44792010040615005, + "pass@32": 0.49754141889425796, + "pass@64": 0.5335500754567829, + "pass@128": 0.562874251497006 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 598532, + "answer_accuracy": 0.7470891915641477, + "avg_response_len": 296.05446906454705, + "resp_tokens_sum": 237184630, + "pass_at_k": { + "pass@1": 0.7470891915641477, + "pass@2": 0.786041741624341, + "pass@4": 0.8171963422989587, + "pass@8": 0.8418723867323799, + "pass@16": 0.8618735564286889, + "pass@32": 0.8787433244425196, + "pass@64": 0.8934028824733017, + "pass@128": 0.9063748202588273 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9541977611940299, + "pass@2": 0.9678156951463157, + "pass@4": 0.9748611783601676, + "pass@8": 0.9796577407085042, + "pass@16": 0.9838480089184216, + "pass@32": 0.9875043762636851, + "pass@64": 0.9900383081967405, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999512178324649, + "pass@4": 0.9999993495710995, + "pass@8": 0.9999999999504271, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999109100877193, + "pass@2": 0.9997971059538611, + "pass@4": 0.9999872888695931, + "pass@8": 0.9999999628675824, + "pass@16": 0.999999999999931, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.978079044117647, + "pass@2": 0.9832228462251044, + "pass@4": 0.9876504667615077, + "pass@8": 0.9919289408712592, + "pass@16": 0.9954125549449143, + "pass@32": 0.9976171476637306, + "pass@64": 0.9991856523243703, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9926804022082019, + "pass@2": 0.9940603343351797, + "pass@4": 0.9953427139628053, + "pass@8": 0.9967069697195312, + "pass@16": 0.998284717005795, + "pass@32": 0.9995420820514109, + "pass@64": 0.9999790726378381, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.983534414556962, + "pass@2": 0.9882804713196452, + "pass@4": 0.9915554633993535, + "pass@8": 0.9943213356780815, + "pass@16": 0.9969376777404727, + "pass@32": 0.9991500996745862, + "pass@64": 0.9999672926515153, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9854072822822822, + "pass@2": 0.9906026843772905, + "pass@4": 0.9937377819326637, + "pass@8": 0.9952448226037982, + "pass@16": 0.9961299961453437, + "pass@32": 0.9967927486364095, + "pass@64": 0.996992658448461, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9826165594855305, + "pass@2": 0.9903018741929768, + "pass@4": 0.993533330138395, + "pass@8": 0.9951085564879916, + "pass@16": 0.9962574662339553, + "pass@32": 0.9973697274291756, + "pass@64": 0.9983876375026929, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9327628968253968, + "pass@2": 0.9557192069741284, + "pass@4": 0.9682437424488604, + "pass@8": 0.9741818516354278, + "pass@16": 0.9781640472203527, + "pass@32": 0.9821227988899696, + "pass@64": 0.9857483461229213, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8999904434250765, + "pass@2": 0.9349103186929618, + "pass@4": 0.9538260647464932, + "pass@8": 0.9663081643819669, + "pass@16": 0.9746650643954538, + "pass@32": 0.980222122145579, + "pass@64": 0.9853088677588914, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7683497299382716, + "pass@2": 0.8305552517740837, + "pass@4": 0.8701254883648802, + "pass@8": 0.896452794155191, + "pass@16": 0.9145623014547616, + "pass@32": 0.9288403670837605, + "pass@64": 0.9414898958367518, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6926233308605341, + "pass@2": 0.7692027705086564, + "pass@4": 0.8192615537301153, + "pass@8": 0.8520108678448942, + "pass@16": 0.8757373907756257, + "pass@32": 0.8961711597315235, + "pass@64": 0.9141068795826112, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.6027440200617284, + "pass@2": 0.6809554079177604, + "pass@4": 0.7372666376772344, + "pass@8": 0.779929989805882, + "pass@16": 0.8129552551752913, + "pass@32": 0.838654820016913, + "pass@64": 0.8600677120073306, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.3980687684365782, + "pass@2": 0.49995862657654505, + "pass@4": 0.5922309152727587, + "pass@8": 0.66985564993658, + "pass@16": 0.733600568857674, + "pass@32": 0.7836795319827353, + "pass@64": 0.8207437780603994, + "pass@128": 0.8525073746312685 + }, + "17": { + "pass@1": 0.31078219126506024, + "pass@2": 0.3998601443174271, + "pass@4": 0.4776901839642033, + "pass@8": 0.5421539850777513, + "pass@16": 0.5967697115816433, + "pass@32": 0.6470728071560454, + "pass@64": 0.6948854314578804, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.31060987903225806, + "pass@2": 0.3953545688341375, + "pass@4": 0.46816725237974255, + "pass@8": 0.5245444802749871, + "pass@16": 0.5675954269101743, + "pass@32": 0.6024381639921098, + "pass@64": 0.6334866926650284, + "pass@128": 0.6645161290322581 + }, + "19": { + "pass@1": 0.20069391993957703, + "pass@2": 0.27735685229678597, + "pass@4": 0.352942397400476, + "pass@8": 0.4218425693813008, + "pass@16": 0.48200046247383743, + "pass@32": 0.5329621251647273, + "pass@64": 0.5751940364962989, + "pass@128": 0.6072507552870091 + }, + "20": { + "pass@1": 0.19054992492492492, + "pass@2": 0.26671602705067277, + "pass@4": 0.3411375731749749, + "pass@8": 0.40642809177336414, + "pass@16": 0.4636940041024634, + "pass@32": 0.5168443386508894, + "pass@64": 0.5698886737951843, + "pass@128": 0.6246246246246246 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..82b11d7fc37999adcf7461e14e2b7bfc657c7b9a --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8fd4ff218f636d9324dce3b7147a893f79a195b1cbb27562986f747017d37a0 +size 1789327462 diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..39d1328b42e9a0306df0d04a93e57650b386caf6 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op7-10", + "id": { + "answer_accuracy": 0.9948680555555556, + "avg_response_len": 200.86559375, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9948680555555556, + "pass@2": 0.9967000628827644, + "pass@4": 0.9978379090530352, + "pass@8": 0.9985671866533024, + "pass@16": 0.9990748765963088, + "pass@32": 0.9994383140109238, + "pass@64": 0.9996631151246024, + "pass@128": 0.9997777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965390625, + "pass@2": 0.9762990895669291, + "pass@4": 0.9835310340269965, + "pass@8": 0.9884410750465406, + "pass@16": 0.9920800322465261, + "pass@32": 0.9949875707427429, + "pass@64": 0.9969682024594496, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998515625, + "pass@2": 0.9999875738188976, + "pass@4": 0.9999999056992877, + "pass@8": 0.9999999999978996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99715625, + "pass@2": 0.9987378198818899, + "pass@4": 0.9994170550243718, + "pass@8": 0.9997328454284017, + "pass@16": 0.999936088151481, + "pass@32": 0.9999974424774313, + "pass@64": 0.9999999996028359, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997734375, + "pass@2": 0.9999790846456693, + "pass@4": 0.9999998391451069, + "pass@8": 0.9999999999969994, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99934375, + "pass@2": 0.9997969980314959, + "pass@4": 0.9999678398012749, + "pass@8": 0.9999991483688844, + "pass@16": 0.9999999996812912, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.99590625, + "pass@2": 0.9978698326771656, + "pass@4": 0.9991760397450319, + "pass@8": 0.999806357952211, + "pass@16": 0.9999836397392413, + "pass@32": 0.9999999021788922, + "pass@64": 0.9999999999999175, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996390625, + "pass@2": 0.9976301673228347, + "pass@4": 0.9984494680352457, + "pass@8": 0.9991252530887876, + "pass@16": 0.99967412954824, + "pass@32": 0.9999599106992464, + "pass@64": 0.9999998340592174, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.965390625, + "2": 1.0, + "3": 1.0, + "4": 0.9998515625, + "5": 0.99715625, + "6": 0.9997734375, + "7": 0.99934375, + "8": 0.99590625, + "9": 0.996390625 + }, + "per_op_avg_response_len": { + "10": 282.6471796875, + "2": 115.9266484375, + "3": 145.2121484375, + "4": 160.4227421875, + "5": 173.4968984375, + "6": 199.845296875, + "7": 224.9173984375, + "8": 244.6364375, + "9": 260.68559375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.6471796875, + "2": 115.9266484375, + "3": 145.2121484375, + "4": 160.4227421875, + "5": 173.4968984375, + "6": 199.845296875, + "7": 224.9173984375, + "8": 244.6364375, + "9": 260.68559375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377348, + "answer_accuracy": 0.995620145221209, + "avg_response_len": 191.40559302178318, + "resp_tokens_sum": 72544251, + "pass_at_k": { + "pass@1": 0.995620145221209, + "pass@2": 0.9972249147048107, + "pass@4": 0.9982218367270959, + "pass@8": 0.9989679489649705, + "pass@16": 0.9995279150663363, + "pass@32": 0.9998846071668368, + "pass@64": 0.999995670690639, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9708702064896755, + "pass@2": 0.9811478729705245, + "pass@4": 0.9879678469394865, + "pass@8": 0.9929074960686314, + "pass@16": 0.99647263878476, + "pass@32": 0.9990402347500129, + "pass@64": 0.9999622679368442, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9995361328125, + "pass@2": 0.9999611681840552, + "pass@4": 0.9999997053102737, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961061507936508, + "pass@2": 0.9974245406824148, + "pass@4": 0.9983225114717802, + "pass@8": 0.9991563175177627, + "pass@16": 0.9997971079631455, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391619, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9994157823929551, + "pass@4": 0.9999049097228232, + "pass@8": 0.9999974803814076, + "pass@16": 0.9999999990570743, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997245592948718, + "pass@2": 0.9999964510145367, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9969850352112676, + "pass@2": 0.9977355273372518, + "pass@4": 0.9982377527632988, + "pass@8": 0.9989156826284533, + "pass@16": 0.9996108261231167, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388618, + "answer_accuracy": 0.9947831340104849, + "avg_response_len": 198.87665362876803, + "resp_tokens_sum": 77692358, + "pass_at_k": { + "pass@1": 0.9947831340104849, + "pass@2": 0.9967168085855668, + "pass@4": 0.9979331862575991, + "pass@8": 0.99861717442997, + "pass@16": 0.999011118886698, + "pass@32": 0.9993577188566564, + "pass@64": 0.9997158296105277, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9624233128834356, + "pass@2": 0.9752148145017154, + "pass@4": 0.9837717881123756, + "pass@8": 0.9885921198963061, + "pass@16": 0.9913190882562777, + "pass@32": 0.9940677527341384, + "pass@64": 0.9973400318824216, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999116847826087, + "pass@2": 0.9998448733310509, + "pass@4": 0.99999352771121, + "pass@8": 0.9999999912981886, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996642191977078, + "pass@2": 0.9999721504636419, + "pass@4": 0.9999998079573864, + "pass@8": 0.9999999999974207, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997516257225434, + "pass@2": 0.9999839987938646, + "pass@4": 0.9999999431068226, + "pass@8": 0.999999999999909, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9949163732394366, + "pass@2": 0.9973997033381392, + "pass@4": 0.999074047698263, + "pass@8": 0.999839799566731, + "pass@16": 0.9999945768483179, + "pass@32": 0.9999999970489517, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959300149700598, + "pass@2": 0.9971603299306899, + "pass@4": 0.9979446534377815, + "pass@8": 0.9986690402123942, + "pass@16": 0.9994426266152802, + "pass@32": 0.9999211724754741, + "pass@64": 0.9999995855618604, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380122, + "answer_accuracy": 0.9942092818881821, + "avg_response_len": 212.2754723593907, + "resp_tokens_sum": 81160555, + "pass_at_k": { + "pass@1": 0.9942092818881821, + "pass@2": 0.9961626694679571, + "pass@4": 0.9973599727051191, + "pass@8": 0.9981188371724735, + "pass@16": 0.9986909266866945, + "pass@32": 0.9990782545452941, + "pass@64": 0.9992795926465039, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9627332089552239, + "pass@2": 0.9724475555294394, + "pass@4": 0.9788069581787349, + "pass@8": 0.9837743366957562, + "pass@16": 0.9883754773938696, + "pass@32": 0.991781623197491, + "pass@64": 0.9935765469707158, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961397058823529, + "pass@2": 0.9988312007874015, + "pass@4": 0.9998461672070401, + "pass@8": 0.9999959070953863, + "pass@16": 0.9999999974796792, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996549684542587, + "pass@2": 0.9999646818102784, + "pass@4": 0.9999997039999338, + "pass@8": 0.9999999999933739, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993384009009009, + "pass@2": 0.9963785143411128, + "pass@4": 0.9985127714891495, + "pass@8": 0.9995892765946597, + "pass@16": 0.9999566515257909, + "pass@32": 0.9999997093889322, + "pass@64": 0.9999999999997521, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962067926045016, + "pass@2": 0.9980144852773629, + "pass@4": 0.999233284746159, + "pass@8": 0.9998544254815018, + "pass@16": 0.9999950128136015, + "pass@32": 0.9999999978428284, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.49931328125, + "avg_response_len": 345.07323984375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.49931328125, + "pass@2": 0.5720481176181046, + "pass@4": 0.634013232039738, + "pass@8": 0.6852958932021572, + "pass@16": 0.7272454890935137, + "pass@32": 0.7618827024795513, + "pass@64": 0.7911886852160416, + "pass@128": 0.8169 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9403671875, + "pass@2": 0.9526200787401576, + "pass@4": 0.9607115194975627, + "pass@8": 0.9669480663564255, + "pass@16": 0.9717499885435925, + "pass@32": 0.9756214255811415, + "pass@64": 0.9788102449740971, + "pass@128": 0.981 + }, + "12": { + "pass@1": 0.870125, + "pass@2": 0.9032128444881896, + "pass@4": 0.9213206480127483, + "pass@8": 0.9334683419154598, + "pass@16": 0.9428686721946118, + "pass@32": 0.9512553807194407, + "pass@64": 0.959064911583851, + "pass@128": 0.964 + }, + "13": { + "pass@1": 0.7255390625, + "pass@2": 0.7901323818897636, + "pass@4": 0.8335371899137605, + "pass@8": 0.8664799402069285, + "pass@16": 0.8924767976784048, + "pass@32": 0.9119173699096086, + "pass@64": 0.9264492025730127, + "pass@128": 0.939 + }, + "14": { + "pass@1": 0.672890625, + "pass@2": 0.7549088336614174, + "pass@4": 0.8092866258905134, + "pass@8": 0.845706054097624, + "pass@16": 0.8732148229463903, + "pass@32": 0.8953072479891372, + "pass@64": 0.9116997290104822, + "pass@128": 0.923 + }, + "15": { + "pass@1": 0.5484375, + "pass@2": 0.6516536663385825, + "pass@4": 0.72887462007874, + "pass@8": 0.7858731460747265, + "pass@16": 0.8289383732858154, + "pass@32": 0.8619947555520242, + "pass@64": 0.8876994882849462, + "pass@128": 0.909 + }, + "16": { + "pass@1": 0.369890625, + "pass@2": 0.4766964812992127, + "pass@4": 0.5688218582677164, + "pass@8": 0.6408565378603006, + "pass@16": 0.6972583039880819, + "pass@32": 0.7438688002090232, + "pass@64": 0.7830971456841558, + "pass@128": 0.816 + }, + "17": { + "pass@1": 0.2822109375, + "pass@2": 0.3794824064960633, + "pass@4": 0.47264126396700384, + "pass@8": 0.5527894596704014, + "pass@16": 0.6177442607369636, + "pass@32": 0.6714768833872531, + "pass@64": 0.7174887137771621, + "pass@128": 0.756 + }, + "18": { + "pass@1": 0.2381328125, + "pass@2": 0.32302423720472434, + "pass@4": 0.40655892660292464, + "pass@8": 0.4821210774768775, + "pass@16": 0.5467707329228031, + "pass@32": 0.6000760068012647, + "pass@64": 0.6446310090831813, + "pass@128": 0.686 + }, + "19": { + "pass@1": 0.1800703125, + "pass@2": 0.2551035925196851, + "pass@4": 0.3333127756842891, + "pass@8": 0.4052867110946467, + "pass@16": 0.46639368460619895, + "pass@32": 0.5188577412564194, + "pass@64": 0.5681043066447663, + "pass@128": 0.618 + }, + "20": { + "pass@1": 0.16546875, + "pass@2": 0.23364665354330716, + "pass@4": 0.30506689248218943, + "pass@8": 0.37342959726819336, + "pass@16": 0.4350392540321157, + "pass@32": 0.48845141339020415, + "pass@64": 0.5348421005446857, + "pass@128": 0.577 + } + }, + "per_op_accuracy": { + "11": 0.9403671875, + "12": 0.870125, + "13": 0.7255390625, + "14": 0.672890625, + "15": 0.5484375, + "16": 0.369890625, + "17": 0.2822109375, + "18": 0.2381328125, + "19": 0.1800703125, + "20": 0.16546875 + }, + "per_op_avg_response_len": { + "11": 299.526078125, + "12": 309.4031484375, + "13": 325.701265625, + "14": 348.11009375, + "15": 356.8016875, + "16": 358.70609375, + "17": 365.1570390625, + "18": 363.78540625, + "19": 362.2403359375, + "20": 361.30125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.526078125, + "12": 309.4031484375, + "13": 325.701265625, + "14": 348.11009375, + "15": 356.8016875, + "16": 358.70609375, + "17": 365.1570390625, + "18": 363.78540625, + "19": 362.2403359375, + "20": 361.30125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 215252, + "answer_accuracy": 0.4997492570579495, + "avg_response_len": 328.4734096396731, + "resp_tokens_sum": 141480067, + "pass_at_k": { + "pass@1": 0.4997492570579495, + "pass@2": 0.5722816072118052, + "pass@4": 0.6335176211814562, + "pass@8": 0.6836797839343276, + "pass@16": 0.7244547550266026, + "pass@32": 0.7586154112629027, + "pass@64": 0.7886576024604199, + "pass@128": 0.8148588410104012 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9255709985835694, + "pass@2": 0.9379311330329462, + "pass@4": 0.9460841846787563, + "pass@8": 0.953538343339806, + "pass@16": 0.9606344539129097, + "pass@32": 0.9667070067195095, + "pass@64": 0.9711033125075582, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8532618290960452, + "pass@2": 0.8889477401129945, + "pass@4": 0.908801012956007, + "pass@8": 0.9212338101619112, + "pass@16": 0.9299795536740266, + "pass@32": 0.9375541842746691, + "pass@64": 0.94540869716572, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.7385873824451411, + "pass@2": 0.8001004307259397, + "pass@4": 0.8404669673931826, + "pass@8": 0.8699546007905485, + "pass@16": 0.8925697001671176, + "pass@32": 0.9083751965325955, + "pass@64": 0.9196466022954688, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682948792834891, + "pass@2": 0.7658664636838621, + "pass@4": 0.8201422071072896, + "pass@8": 0.8560071062268605, + "pass@16": 0.881615389631223, + "pass@32": 0.9003916850121438, + "pass@64": 0.9136442133530225, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5601968930635838, + "pass@2": 0.6652793028310041, + "pass@4": 0.7403421975612876, + "pass@8": 0.7962299675301491, + "pass@16": 0.841140935305107, + "pass@32": 0.8766070377457041, + "pass@64": 0.9036559422239621, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.3536290322580645, + "pass@2": 0.4650046831343661, + "pass@4": 0.5630961582423161, + "pass@8": 0.6376798757436568, + "pass@16": 0.6915151521482109, + "pass@32": 0.7342275627875268, + "pass@64": 0.771219279575759, + "pass@128": 0.8 + }, + "17": { + "pass@1": 0.27966943641618497, + "pass@2": 0.37408579775613293, + "pass@4": 0.4624504658196626, + "pass@8": 0.5392494778039917, + "pass@16": 0.6049840411673603, + "pass@32": 0.6638390199370979, + "pass@64": 0.7175498472501749, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.22254464285714284, + "pass@2": 0.30653648762654667, + "pass@4": 0.39112811853875373, + "pass@8": 0.4671507144054809, + "pass@16": 0.5302606269042149, + "pass@32": 0.5815471929725439, + "pass@64": 0.626977580595557, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.18036786786786788, + "pass@2": 0.25607694308481704, + "pass@4": 0.3336125036735274, + "pass@8": 0.4036587555938847, + "pass@16": 0.46222010352371423, + "pass@32": 0.5145560690001918, + "pass@64": 0.5671466447500417, + "pass@128": 0.6126126126126126 + }, + "20": { + "pass@1": 0.17978134384384384, + "pass@2": 0.2478741044193799, + "pass@4": 0.31836346132409143, + "pass@8": 0.38406938429696535, + "pass@16": 0.44296035763490194, + "pass@32": 0.4964207710865561, + "pass@64": 0.5447281075440609, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 216658, + "answer_accuracy": 0.503312704430568, + "avg_response_len": 341.47653230002976, + "resp_tokens_sum": 146993354, + "pass_at_k": { + "pass@1": 0.503312704430568, + "pass@2": 0.5763784137124492, + "pass@4": 0.6386902670841463, + "pass@8": 0.6901693132140736, + "pass@16": 0.7319171255952822, + "pass@32": 0.7661119720921368, + "pass@64": 0.7951950586913296, + "pass@128": 0.8212905144216474 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9519248870481928, + "pass@2": 0.9623012226069635, + "pass@4": 0.9677496744571086, + "pass@8": 0.9707849085829278, + "pass@16": 0.972937649452467, + "pass@32": 0.9755969290692811, + "pass@64": 0.9793188502021436, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8681916144200627, + "pass@2": 0.8977253443339176, + "pass@4": 0.9139599540653028, + "pass@8": 0.9263390661312996, + "pass@16": 0.937316041001789, + "pass@32": 0.9479035214285512, + "pass@64": 0.9577698170870994, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.7415966386554622, + "pass@2": 0.8074346864730148, + "pass@4": 0.8499324283519178, + "pass@8": 0.8808579490271415, + "pass@16": 0.9039888966185219, + "pass@32": 0.9209711586353112, + "pass@64": 0.9340631262129528, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6876599049707602, + "pass@2": 0.7685266726527606, + "pass@4": 0.8217676359533996, + "pass@8": 0.8564829892788888, + "pass@16": 0.882334674211943, + "pass@32": 0.9022700593687277, + "pass@64": 0.9155857627723111, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5451704545454545, + "pass@2": 0.654227063946552, + "pass@4": 0.7373655077206254, + "pass@8": 0.7962415469915555, + "pass@16": 0.8375704897171268, + "pass@32": 0.8683757778577055, + "pass@64": 0.8930159138899741, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.3810763888888889, + "pass@2": 0.4868089906229668, + "pass@4": 0.5760386284513578, + "pass@8": 0.6432655475632902, + "pass@16": 0.6953101222141723, + "pass@32": 0.7400005671118122, + "pass@64": 0.7801792827865532, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.29670516304347827, + "pass@2": 0.3995455476353497, + "pass@4": 0.5012611435022484, + "pass@8": 0.590376899556164, + "pass@16": 0.6594289276417519, + "pass@32": 0.711161429942676, + "pass@64": 0.7534745261944554, + "pass@128": 0.7857142857142857 + }, + "18": { + "pass@1": 0.23887867647058825, + "pass@2": 0.3285324513663731, + "pass@4": 0.4181968639030413, + "pass@8": 0.5009363789683653, + "pass@16": 0.5731285778604139, + "pass@32": 0.6325426409355972, + "pass@64": 0.6800289019744267, + "pass@128": 0.7205882352941176 + }, + "19": { + "pass@1": 0.17503720238095238, + "pass@2": 0.24297329044806892, + "pass@4": 0.31580475850116946, + "pass@8": 0.3867082039757981, + "pass@16": 0.44867577405752523, + "pass@32": 0.5014357515012002, + "pass@64": 0.5501519038964435, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.1511508233532934, + "pass@2": 0.217524842048187, + "pass@4": 0.28555462752784627, + "pass@8": 0.3501604003392409, + "pass@16": 0.4087684917529828, + "pass@32": 0.4609191551164828, + "pass@64": 0.5082929397545116, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 207211, + "answer_accuracy": 0.49475425962713937, + "avg_response_len": 365.84162496179704, + "resp_tokens_sum": 153220326, + "pass_at_k": { + "pass@1": 0.49475425962713937, + "pass@2": 0.5673572624078336, + "pass@4": 0.6297158181289402, + "pass@8": 0.6819489788336432, + "pass@16": 0.7253139813244391, + "pass@32": 0.7608959669162503, + "pass@64": 0.7896739112170068, + "pass@128": 0.8144865525672371 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.944766865079365, + "pass@2": 0.9588774059492565, + "pass@4": 0.9696854297974654, + "pass@8": 0.9779315603426709, + "pass@16": 0.9829546879178315, + "pass@32": 0.9856370532004864, + "pass@64": 0.9869108488629128, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.890266628440367, + "pass@2": 0.9240090509041881, + "pass@4": 0.9420546302125073, + "pass@8": 0.9536679236154701, + "pass@16": 0.9622388168637205, + "pass@32": 0.9693577252308863, + "pass@64": 0.9751121135669748, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6949990354938271, + "pass@2": 0.7612535846213664, + "pass@4": 0.8086492296101871, + "pass@8": 0.8472164652841165, + "pass@16": 0.8797007013342069, + "pass@32": 0.9054289462435336, + "pass@64": 0.9247574085886842, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.648321587537092, + "pass@2": 0.7306515631206338, + "pass@4": 0.7862802549347504, + "pass@8": 0.8249572423306274, + "pass@16": 0.8559579355913478, + "pass@32": 0.883398091383188, + "pass@64": 0.9059038743502413, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5392071759259259, + "pass@2": 0.6344817791873233, + "pass@4": 0.7079802536257039, + "pass@8": 0.7642526753150662, + "pass@16": 0.8071152718012273, + "pass@32": 0.8398910919721222, + "pass@64": 0.865244693493098, + "pass@128": 0.8858024691358025 + }, + "16": { + "pass@1": 0.37317938790560473, + "pass@2": 0.47691762188465375, + "pass@4": 0.5665855180713028, + "pass@8": 0.6412671657376158, + "pass@16": 0.7045272977137541, + "pass@32": 0.7566904327098634, + "pass@64": 0.7969800612318285, + "pass@128": 0.831858407079646 + }, + "17": { + "pass@1": 0.2708019578313253, + "pass@2": 0.36564775341523553, + "pass@4": 0.45550395959541135, + "pass@8": 0.5304451165455887, + "pass@16": 0.5906134572060683, + "pass@32": 0.6409475965285415, + "pass@64": 0.6825230999818859, + "pass@128": 0.7168674698795181 + }, + "18": { + "pass@1": 0.25491431451612906, + "pass@2": 0.33559817119634233, + "pass@4": 0.4112166173784727, + "pass@8": 0.478386963179724, + "pass@16": 0.5365025710767365, + "pass@32": 0.5853870688476496, + "pass@64": 0.6257388038820341, + "pass@128": 0.6612903225806451 + }, + "19": { + "pass@1": 0.1848800981873112, + "pass@2": 0.26643789994528644, + "pass@4": 0.3507837253915768, + "pass@8": 0.4257836524048795, + "pass@16": 0.4885780364032442, + "pass@32": 0.540870567295929, + "pass@64": 0.5872913420960638, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1655170795795796, + "pass@2": 0.2355894280106878, + "pass@4": 0.31134118398488475, + "pass@8": 0.38612888463663003, + "pass@16": 0.4534678038864786, + "pass@32": 0.5080969934218494, + "pass@64": 0.5515849815450634, + "pass@128": 0.5915915915915916 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7340497532894736, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7340497532894736, + "pass@2": 0.7731990390592433, + "pass@4": 0.8063512369407585, + "pass@8": 0.8336875585211297, + "pass@16": 0.8560067779106318, + "pass@32": 0.8744090447838998, + "pass@64": 0.8899397309622028, + "pass@128": 0.9035263157894737 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965390625, + "pass@2": 0.9762990895669291, + "pass@4": 0.9835310340269965, + "pass@8": 0.9884410750465406, + "pass@16": 0.9920800322465261, + "pass@32": 0.9949875707427429, + "pass@64": 0.9969682024594496, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998515625, + "pass@2": 0.9999875738188976, + "pass@4": 0.9999999056992877, + "pass@8": 0.9999999999978996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99715625, + "pass@2": 0.9987378198818899, + "pass@4": 0.9994170550243718, + "pass@8": 0.9997328454284017, + "pass@16": 0.999936088151481, + "pass@32": 0.9999974424774313, + "pass@64": 0.9999999996028359, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997734375, + "pass@2": 0.9999790846456693, + "pass@4": 0.9999998391451069, + "pass@8": 0.9999999999969994, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99934375, + "pass@2": 0.9997969980314959, + "pass@4": 0.9999678398012749, + "pass@8": 0.9999991483688844, + "pass@16": 0.9999999996812912, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.99590625, + "pass@2": 0.9978698326771656, + "pass@4": 0.9991760397450319, + "pass@8": 0.999806357952211, + "pass@16": 0.9999836397392413, + "pass@32": 0.9999999021788922, + "pass@64": 0.9999999999999175, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996390625, + "pass@2": 0.9976301673228347, + "pass@4": 0.9984494680352457, + "pass@8": 0.9991252530887876, + "pass@16": 0.99967412954824, + "pass@32": 0.9999599106992464, + "pass@64": 0.9999998340592174, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9403671875, + "pass@2": 0.9526200787401576, + "pass@4": 0.9607115194975627, + "pass@8": 0.9669480663564255, + "pass@16": 0.9717499885435925, + "pass@32": 0.9756214255811415, + "pass@64": 0.9788102449740971, + "pass@128": 0.981 + }, + "12": { + "pass@1": 0.870125, + "pass@2": 0.9032128444881896, + "pass@4": 0.9213206480127483, + "pass@8": 0.9334683419154598, + "pass@16": 0.9428686721946118, + "pass@32": 0.9512553807194407, + "pass@64": 0.959064911583851, + "pass@128": 0.964 + }, + "13": { + "pass@1": 0.7255390625, + "pass@2": 0.7901323818897636, + "pass@4": 0.8335371899137605, + "pass@8": 0.8664799402069285, + "pass@16": 0.8924767976784048, + "pass@32": 0.9119173699096086, + "pass@64": 0.9264492025730127, + "pass@128": 0.939 + }, + "14": { + "pass@1": 0.672890625, + "pass@2": 0.7549088336614174, + "pass@4": 0.8092866258905134, + "pass@8": 0.845706054097624, + "pass@16": 0.8732148229463903, + "pass@32": 0.8953072479891372, + "pass@64": 0.9116997290104822, + "pass@128": 0.923 + }, + "15": { + "pass@1": 0.5484375, + "pass@2": 0.6516536663385825, + "pass@4": 0.72887462007874, + "pass@8": 0.7858731460747265, + "pass@16": 0.8289383732858154, + "pass@32": 0.8619947555520242, + "pass@64": 0.8876994882849462, + "pass@128": 0.909 + }, + "16": { + "pass@1": 0.369890625, + "pass@2": 0.4766964812992127, + "pass@4": 0.5688218582677164, + "pass@8": 0.6408565378603006, + "pass@16": 0.6972583039880819, + "pass@32": 0.7438688002090232, + "pass@64": 0.7830971456841558, + "pass@128": 0.816 + }, + "17": { + "pass@1": 0.2822109375, + "pass@2": 0.3794824064960633, + "pass@4": 0.47264126396700384, + "pass@8": 0.5527894596704014, + "pass@16": 0.6177442607369636, + "pass@32": 0.6714768833872531, + "pass@64": 0.7174887137771621, + "pass@128": 0.756 + }, + "18": { + "pass@1": 0.2381328125, + "pass@2": 0.32302423720472434, + "pass@4": 0.40655892660292464, + "pass@8": 0.4821210774768775, + "pass@16": 0.5467707329228031, + "pass@32": 0.6000760068012647, + "pass@64": 0.6446310090831813, + "pass@128": 0.686 + }, + "19": { + "pass@1": 0.1800703125, + "pass@2": 0.2551035925196851, + "pass@4": 0.3333127756842891, + "pass@8": 0.4052867110946467, + "pass@16": 0.46639368460619895, + "pass@32": 0.5188577412564194, + "pass@64": 0.5681043066447663, + "pass@128": 0.618 + }, + "20": { + "pass@1": 0.16546875, + "pass@2": 0.23364665354330716, + "pass@4": 0.30506689248218943, + "pass@8": 0.37342959726819336, + "pass@16": 0.4350392540321157, + "pass@32": 0.48845141339020415, + "pass@64": 0.5348421005446857, + "pass@128": 0.577 + } + }, + "per_op_accuracy": { + "10": 0.965390625, + "2": 1.0, + "3": 1.0, + "4": 0.9998515625, + "5": 0.99715625, + "6": 0.9997734375, + "7": 0.99934375, + "8": 0.99590625, + "9": 0.996390625, + "11": 0.9403671875, + "12": 0.870125, + "13": 0.7255390625, + "14": 0.672890625, + "15": 0.5484375, + "16": 0.369890625, + "17": 0.2822109375, + "18": 0.2381328125, + "19": 0.1800703125, + "20": 0.16546875 + }, + "per_op_avg_response_len": { + "10": 282.6471796875, + "2": 115.9266484375, + "3": 145.2121484375, + "4": 160.4227421875, + "5": 173.4968984375, + "6": 199.845296875, + "7": 224.9173984375, + "8": 244.6364375, + "9": 260.68559375, + "11": 299.526078125, + "12": 309.4031484375, + "13": 325.701265625, + "14": 348.11009375, + "15": 356.8016875, + "16": 358.70609375, + "17": 365.1570390625, + "18": 363.78540625, + "19": 362.2403359375, + "20": 361.30125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.6471796875, + "2": 115.9266484375, + "3": 145.2121484375, + "4": 160.4227421875, + "5": 173.4968984375, + "6": 199.845296875, + "7": 224.9173984375, + "8": 244.6364375, + "9": 260.68559375, + "11": 299.526078125, + "12": 309.4031484375, + "13": 325.701265625, + "14": 348.11009375, + "15": 356.8016875, + "16": 358.70609375, + "17": 365.1570390625, + "18": 363.78540625, + "19": 362.2403359375, + "20": 361.30125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 592600, + "answer_accuracy": 0.7318506955422067, + "avg_response_len": 264.31631115633894, + "resp_tokens_sum": 214024318, + "pass_at_k": { + "pass@1": 0.7318506955422067, + "pass@2": 0.7711840943263716, + "pass@4": 0.8042240995612536, + "pass@8": 0.8312561760708604, + "pass@16": 0.8532077785608563, + "pass@32": 0.8715458711224592, + "pass@64": 0.8875782505839926, + "pass@128": 0.9015175466329434 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9708702064896755, + "pass@2": 0.9811478729705245, + "pass@4": 0.9879678469394865, + "pass@8": 0.9929074960686314, + "pass@16": 0.99647263878476, + "pass@32": 0.9990402347500129, + "pass@64": 0.9999622679368442, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9995361328125, + "pass@2": 0.9999611681840552, + "pass@4": 0.9999997053102737, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961061507936508, + "pass@2": 0.9974245406824148, + "pass@4": 0.9983225114717802, + "pass@8": 0.9991563175177627, + "pass@16": 0.9997971079631455, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391619, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9994157823929551, + "pass@4": 0.9999049097228232, + "pass@8": 0.9999974803814076, + "pass@16": 0.9999999990570743, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997245592948718, + "pass@2": 0.9999964510145367, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9969850352112676, + "pass@2": 0.9977355273372518, + "pass@4": 0.9982377527632988, + "pass@8": 0.9989156826284533, + "pass@16": 0.9996108261231167, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9255709985835694, + "pass@2": 0.9379311330329462, + "pass@4": 0.9460841846787563, + "pass@8": 0.953538343339806, + "pass@16": 0.9606344539129097, + "pass@32": 0.9667070067195095, + "pass@64": 0.9711033125075582, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8532618290960452, + "pass@2": 0.8889477401129945, + "pass@4": 0.908801012956007, + "pass@8": 0.9212338101619112, + "pass@16": 0.9299795536740266, + "pass@32": 0.9375541842746691, + "pass@64": 0.94540869716572, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.7385873824451411, + "pass@2": 0.8001004307259397, + "pass@4": 0.8404669673931826, + "pass@8": 0.8699546007905485, + "pass@16": 0.8925697001671176, + "pass@32": 0.9083751965325955, + "pass@64": 0.9196466022954688, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682948792834891, + "pass@2": 0.7658664636838621, + "pass@4": 0.8201422071072896, + "pass@8": 0.8560071062268605, + "pass@16": 0.881615389631223, + "pass@32": 0.9003916850121438, + "pass@64": 0.9136442133530225, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5601968930635838, + "pass@2": 0.6652793028310041, + "pass@4": 0.7403421975612876, + "pass@8": 0.7962299675301491, + "pass@16": 0.841140935305107, + "pass@32": 0.8766070377457041, + "pass@64": 0.9036559422239621, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.3536290322580645, + "pass@2": 0.4650046831343661, + "pass@4": 0.5630961582423161, + "pass@8": 0.6376798757436568, + "pass@16": 0.6915151521482109, + "pass@32": 0.7342275627875268, + "pass@64": 0.771219279575759, + "pass@128": 0.8 + }, + "17": { + "pass@1": 0.27966943641618497, + "pass@2": 0.37408579775613293, + "pass@4": 0.4624504658196626, + "pass@8": 0.5392494778039917, + "pass@16": 0.6049840411673603, + "pass@32": 0.6638390199370979, + "pass@64": 0.7175498472501749, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.22254464285714284, + "pass@2": 0.30653648762654667, + "pass@4": 0.39112811853875373, + "pass@8": 0.4671507144054809, + "pass@16": 0.5302606269042149, + "pass@32": 0.5815471929725439, + "pass@64": 0.626977580595557, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.18036786786786788, + "pass@2": 0.25607694308481704, + "pass@4": 0.3336125036735274, + "pass@8": 0.4036587555938847, + "pass@16": 0.46222010352371423, + "pass@32": 0.5145560690001918, + "pass@64": 0.5671466447500417, + "pass@128": 0.6126126126126126 + }, + "20": { + "pass@1": 0.17978134384384384, + "pass@2": 0.2478741044193799, + "pass@4": 0.31836346132409143, + "pass@8": 0.38406938429696535, + "pass@16": 0.44296035763490194, + "pass@32": 0.4964207710865561, + "pass@64": 0.5447281075440609, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605276, + "answer_accuracy": 0.7371346453624318, + "avg_response_len": 273.6332229150429, + "resp_tokens_sum": 224685712, + "pass_at_k": { + "pass@1": 0.7371346453624318, + "pass@2": 0.7763585822475559, + "pass@4": 0.8096036559099176, + "pass@8": 0.8369164484332293, + "pass@16": 0.8589897471892718, + "pass@32": 0.8770809540290541, + "pass@64": 0.8924978479111944, + "pass@128": 0.9063133281371785 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9624233128834356, + "pass@2": 0.9752148145017154, + "pass@4": 0.9837717881123756, + "pass@8": 0.9885921198963061, + "pass@16": 0.9913190882562777, + "pass@32": 0.9940677527341384, + "pass@64": 0.9973400318824216, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999116847826087, + "pass@2": 0.9998448733310509, + "pass@4": 0.99999352771121, + "pass@8": 0.9999999912981886, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996642191977078, + "pass@2": 0.9999721504636419, + "pass@4": 0.9999998079573864, + "pass@8": 0.9999999999974207, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997516257225434, + "pass@2": 0.9999839987938646, + "pass@4": 0.9999999431068226, + "pass@8": 0.999999999999909, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9949163732394366, + "pass@2": 0.9973997033381392, + "pass@4": 0.999074047698263, + "pass@8": 0.999839799566731, + "pass@16": 0.9999945768483179, + "pass@32": 0.9999999970489517, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959300149700598, + "pass@2": 0.9971603299306899, + "pass@4": 0.9979446534377815, + "pass@8": 0.9986690402123942, + "pass@16": 0.9994426266152802, + "pass@32": 0.9999211724754741, + "pass@64": 0.9999995855618604, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9519248870481928, + "pass@2": 0.9623012226069635, + "pass@4": 0.9677496744571086, + "pass@8": 0.9707849085829278, + "pass@16": 0.972937649452467, + "pass@32": 0.9755969290692811, + "pass@64": 0.9793188502021436, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8681916144200627, + "pass@2": 0.8977253443339176, + "pass@4": 0.9139599540653028, + "pass@8": 0.9263390661312996, + "pass@16": 0.937316041001789, + "pass@32": 0.9479035214285512, + "pass@64": 0.9577698170870994, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.7415966386554622, + "pass@2": 0.8074346864730148, + "pass@4": 0.8499324283519178, + "pass@8": 0.8808579490271415, + "pass@16": 0.9039888966185219, + "pass@32": 0.9209711586353112, + "pass@64": 0.9340631262129528, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6876599049707602, + "pass@2": 0.7685266726527606, + "pass@4": 0.8217676359533996, + "pass@8": 0.8564829892788888, + "pass@16": 0.882334674211943, + "pass@32": 0.9022700593687277, + "pass@64": 0.9155857627723111, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5451704545454545, + "pass@2": 0.654227063946552, + "pass@4": 0.7373655077206254, + "pass@8": 0.7962415469915555, + "pass@16": 0.8375704897171268, + "pass@32": 0.8683757778577055, + "pass@64": 0.8930159138899741, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.3810763888888889, + "pass@2": 0.4868089906229668, + "pass@4": 0.5760386284513578, + "pass@8": 0.6432655475632902, + "pass@16": 0.6953101222141723, + "pass@32": 0.7400005671118122, + "pass@64": 0.7801792827865532, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.29670516304347827, + "pass@2": 0.3995455476353497, + "pass@4": 0.5012611435022484, + "pass@8": 0.590376899556164, + "pass@16": 0.6594289276417519, + "pass@32": 0.711161429942676, + "pass@64": 0.7534745261944554, + "pass@128": 0.7857142857142857 + }, + "18": { + "pass@1": 0.23887867647058825, + "pass@2": 0.3285324513663731, + "pass@4": 0.4181968639030413, + "pass@8": 0.5009363789683653, + "pass@16": 0.5731285778604139, + "pass@32": 0.6325426409355972, + "pass@64": 0.6800289019744267, + "pass@128": 0.7205882352941176 + }, + "19": { + "pass@1": 0.17503720238095238, + "pass@2": 0.24297329044806892, + "pass@4": 0.31580475850116946, + "pass@8": 0.3867082039757981, + "pass@16": 0.44867577405752523, + "pass@32": 0.5014357515012002, + "pass@64": 0.5501519038964435, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.1511508233532934, + "pass@2": 0.217524842048187, + "pass@4": 0.28555462752784627, + "pass@8": 0.3501604003392409, + "pass@16": 0.4087684917529828, + "pass@32": 0.4609191551164828, + "pass@64": 0.5082929397545116, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587333, + "answer_accuracy": 0.7331105707780796, + "avg_response_len": 292.55482230587955, + "resp_tokens_sum": 234380881, + "pass_at_k": { + "pass@1": 0.7331105707780796, + "pass@2": 0.7719972609521001, + "pass@4": 0.8051676618290551, + "pass@8": 0.8328356007953099, + "pass@16": 0.8557784222570307, + "pass@32": 0.8745643633290906, + "pass@64": 0.8897046142733968, + "pass@128": 0.9027001118389519 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9627332089552239, + "pass@2": 0.9724475555294394, + "pass@4": 0.9788069581787349, + "pass@8": 0.9837743366957562, + "pass@16": 0.9883754773938696, + "pass@32": 0.991781623197491, + "pass@64": 0.9935765469707158, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961397058823529, + "pass@2": 0.9988312007874015, + "pass@4": 0.9998461672070401, + "pass@8": 0.9999959070953863, + "pass@16": 0.9999999974796792, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996549684542587, + "pass@2": 0.9999646818102784, + "pass@4": 0.9999997039999338, + "pass@8": 0.9999999999933739, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993384009009009, + "pass@2": 0.9963785143411128, + "pass@4": 0.9985127714891495, + "pass@8": 0.9995892765946597, + "pass@16": 0.9999566515257909, + "pass@32": 0.9999997093889322, + "pass@64": 0.9999999999997521, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962067926045016, + "pass@2": 0.9980144852773629, + "pass@4": 0.999233284746159, + "pass@8": 0.9998544254815018, + "pass@16": 0.9999950128136015, + "pass@32": 0.9999999978428284, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.944766865079365, + "pass@2": 0.9588774059492565, + "pass@4": 0.9696854297974654, + "pass@8": 0.9779315603426709, + "pass@16": 0.9829546879178315, + "pass@32": 0.9856370532004864, + "pass@64": 0.9869108488629128, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.890266628440367, + "pass@2": 0.9240090509041881, + "pass@4": 0.9420546302125073, + "pass@8": 0.9536679236154701, + "pass@16": 0.9622388168637205, + "pass@32": 0.9693577252308863, + "pass@64": 0.9751121135669748, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6949990354938271, + "pass@2": 0.7612535846213664, + "pass@4": 0.8086492296101871, + "pass@8": 0.8472164652841165, + "pass@16": 0.8797007013342069, + "pass@32": 0.9054289462435336, + "pass@64": 0.9247574085886842, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.648321587537092, + "pass@2": 0.7306515631206338, + "pass@4": 0.7862802549347504, + "pass@8": 0.8249572423306274, + "pass@16": 0.8559579355913478, + "pass@32": 0.883398091383188, + "pass@64": 0.9059038743502413, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5392071759259259, + "pass@2": 0.6344817791873233, + "pass@4": 0.7079802536257039, + "pass@8": 0.7642526753150662, + "pass@16": 0.8071152718012273, + "pass@32": 0.8398910919721222, + "pass@64": 0.865244693493098, + "pass@128": 0.8858024691358025 + }, + "16": { + "pass@1": 0.37317938790560473, + "pass@2": 0.47691762188465375, + "pass@4": 0.5665855180713028, + "pass@8": 0.6412671657376158, + "pass@16": 0.7045272977137541, + "pass@32": 0.7566904327098634, + "pass@64": 0.7969800612318285, + "pass@128": 0.831858407079646 + }, + "17": { + "pass@1": 0.2708019578313253, + "pass@2": 0.36564775341523553, + "pass@4": 0.45550395959541135, + "pass@8": 0.5304451165455887, + "pass@16": 0.5906134572060683, + "pass@32": 0.6409475965285415, + "pass@64": 0.6825230999818859, + "pass@128": 0.7168674698795181 + }, + "18": { + "pass@1": 0.25491431451612906, + "pass@2": 0.33559817119634233, + "pass@4": 0.4112166173784727, + "pass@8": 0.478386963179724, + "pass@16": 0.5365025710767365, + "pass@32": 0.5853870688476496, + "pass@64": 0.6257388038820341, + "pass@128": 0.6612903225806451 + }, + "19": { + "pass@1": 0.1848800981873112, + "pass@2": 0.26643789994528644, + "pass@4": 0.3507837253915768, + "pass@8": 0.4257836524048795, + "pass@16": 0.4885780364032442, + "pass@32": 0.540870567295929, + "pass@64": 0.5872913420960638, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1655170795795796, + "pass@2": 0.2355894280106878, + "pass@4": 0.31134118398488475, + "pass@8": 0.38612888463663003, + "pass@16": 0.4534678038864786, + "pass@32": 0.5080969934218494, + "pass@64": 0.5515849815450634, + "pass@128": 0.5915915915915916 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..62571c61b446c767063f7428309d71e3469dbce7 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bcd7af7e4e46cd1ed7d44c94f26662f11c4a793e2af922fc8bd50458d28b959 +size 2614269674 diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/summary.csv b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d1bb633ead673e1f5e54a90ef6ea8cdf47e4335a --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op7-10,0.9948680555555556,0.49931328125,0.7340497532894736,0.0,0.0,0.0,200.86559375,345.07323984375,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/summary.json b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e506ba0dd5d8c2ece59e1d65fcf6063757ec91e1 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op7-10", + "id_acc": 0.9948680555555556, + "ood_acc": 0.49931328125, + "total_acc": 0.7340497532894736, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.86559375, + "ood_avg_resp_len": 345.07323984375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 592600, + "answer_accuracy": 0.7318506955422067, + "avg_response_len": 264.31631115633894, + "resp_tokens_sum": 214024318, + "pass_at_k": { + "pass@1": 0.7318506955422067, + "pass@2": 0.7711840943263716, + "pass@4": 0.8042240995612536, + "pass@8": 0.8312561760708604, + "pass@16": 0.8532077785608563, + "pass@32": 0.8715458711224592, + "pass@64": 0.8875782505839926, + "pass@128": 0.9015175466329434 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9708702064896755, + "pass@2": 0.9811478729705245, + "pass@4": 0.9879678469394865, + "pass@8": 0.9929074960686314, + "pass@16": 0.99647263878476, + "pass@32": 0.9990402347500129, + "pass@64": 0.9999622679368442, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9995361328125, + "pass@2": 0.9999611681840552, + "pass@4": 0.9999997053102737, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961061507936508, + "pass@2": 0.9974245406824148, + "pass@4": 0.9983225114717802, + "pass@8": 0.9991563175177627, + "pass@16": 0.9997971079631455, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391619, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9994157823929551, + "pass@4": 0.9999049097228232, + "pass@8": 0.9999974803814076, + "pass@16": 0.9999999990570743, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997245592948718, + "pass@2": 0.9999964510145367, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9969850352112676, + "pass@2": 0.9977355273372518, + "pass@4": 0.9982377527632988, + "pass@8": 0.9989156826284533, + "pass@16": 0.9996108261231167, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9255709985835694, + "pass@2": 0.9379311330329462, + "pass@4": 0.9460841846787563, + "pass@8": 0.953538343339806, + "pass@16": 0.9606344539129097, + "pass@32": 0.9667070067195095, + "pass@64": 0.9711033125075582, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8532618290960452, + "pass@2": 0.8889477401129945, + "pass@4": 0.908801012956007, + "pass@8": 0.9212338101619112, + "pass@16": 0.9299795536740266, + "pass@32": 0.9375541842746691, + "pass@64": 0.94540869716572, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.7385873824451411, + "pass@2": 0.8001004307259397, + "pass@4": 0.8404669673931826, + "pass@8": 0.8699546007905485, + "pass@16": 0.8925697001671176, + "pass@32": 0.9083751965325955, + "pass@64": 0.9196466022954688, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682948792834891, + "pass@2": 0.7658664636838621, + "pass@4": 0.8201422071072896, + "pass@8": 0.8560071062268605, + "pass@16": 0.881615389631223, + "pass@32": 0.9003916850121438, + "pass@64": 0.9136442133530225, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5601968930635838, + "pass@2": 0.6652793028310041, + "pass@4": 0.7403421975612876, + "pass@8": 0.7962299675301491, + "pass@16": 0.841140935305107, + "pass@32": 0.8766070377457041, + "pass@64": 0.9036559422239621, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.3536290322580645, + "pass@2": 0.4650046831343661, + "pass@4": 0.5630961582423161, + "pass@8": 0.6376798757436568, + "pass@16": 0.6915151521482109, + "pass@32": 0.7342275627875268, + "pass@64": 0.771219279575759, + "pass@128": 0.8 + }, + "17": { + "pass@1": 0.27966943641618497, + "pass@2": 0.37408579775613293, + "pass@4": 0.4624504658196626, + "pass@8": 0.5392494778039917, + "pass@16": 0.6049840411673603, + "pass@32": 0.6638390199370979, + "pass@64": 0.7175498472501749, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.22254464285714284, + "pass@2": 0.30653648762654667, + "pass@4": 0.39112811853875373, + "pass@8": 0.4671507144054809, + "pass@16": 0.5302606269042149, + "pass@32": 0.5815471929725439, + "pass@64": 0.626977580595557, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.18036786786786788, + "pass@2": 0.25607694308481704, + "pass@4": 0.3336125036735274, + "pass@8": 0.4036587555938847, + "pass@16": 0.46222010352371423, + "pass@32": 0.5145560690001918, + "pass@64": 0.5671466447500417, + "pass@128": 0.6126126126126126 + }, + "20": { + "pass@1": 0.17978134384384384, + "pass@2": 0.2478741044193799, + "pass@4": 0.31836346132409143, + "pass@8": 0.38406938429696535, + "pass@16": 0.44296035763490194, + "pass@32": 0.4964207710865561, + "pass@64": 0.5447281075440609, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605276, + "answer_accuracy": 0.7371346453624318, + "avg_response_len": 273.6332229150429, + "resp_tokens_sum": 224685712, + "pass_at_k": { + "pass@1": 0.7371346453624318, + "pass@2": 0.7763585822475559, + "pass@4": 0.8096036559099176, + "pass@8": 0.8369164484332293, + "pass@16": 0.8589897471892718, + "pass@32": 0.8770809540290541, + "pass@64": 0.8924978479111944, + "pass@128": 0.9063133281371785 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9624233128834356, + "pass@2": 0.9752148145017154, + "pass@4": 0.9837717881123756, + "pass@8": 0.9885921198963061, + "pass@16": 0.9913190882562777, + "pass@32": 0.9940677527341384, + "pass@64": 0.9973400318824216, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999116847826087, + "pass@2": 0.9998448733310509, + "pass@4": 0.99999352771121, + "pass@8": 0.9999999912981886, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996642191977078, + "pass@2": 0.9999721504636419, + "pass@4": 0.9999998079573864, + "pass@8": 0.9999999999974207, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997516257225434, + "pass@2": 0.9999839987938646, + "pass@4": 0.9999999431068226, + "pass@8": 0.999999999999909, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9949163732394366, + "pass@2": 0.9973997033381392, + "pass@4": 0.999074047698263, + "pass@8": 0.999839799566731, + "pass@16": 0.9999945768483179, + "pass@32": 0.9999999970489517, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959300149700598, + "pass@2": 0.9971603299306899, + "pass@4": 0.9979446534377815, + "pass@8": 0.9986690402123942, + "pass@16": 0.9994426266152802, + "pass@32": 0.9999211724754741, + "pass@64": 0.9999995855618604, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9519248870481928, + "pass@2": 0.9623012226069635, + "pass@4": 0.9677496744571086, + "pass@8": 0.9707849085829278, + "pass@16": 0.972937649452467, + "pass@32": 0.9755969290692811, + "pass@64": 0.9793188502021436, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8681916144200627, + "pass@2": 0.8977253443339176, + "pass@4": 0.9139599540653028, + "pass@8": 0.9263390661312996, + "pass@16": 0.937316041001789, + "pass@32": 0.9479035214285512, + "pass@64": 0.9577698170870994, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.7415966386554622, + "pass@2": 0.8074346864730148, + "pass@4": 0.8499324283519178, + "pass@8": 0.8808579490271415, + "pass@16": 0.9039888966185219, + "pass@32": 0.9209711586353112, + "pass@64": 0.9340631262129528, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6876599049707602, + "pass@2": 0.7685266726527606, + "pass@4": 0.8217676359533996, + "pass@8": 0.8564829892788888, + "pass@16": 0.882334674211943, + "pass@32": 0.9022700593687277, + "pass@64": 0.9155857627723111, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5451704545454545, + "pass@2": 0.654227063946552, + "pass@4": 0.7373655077206254, + "pass@8": 0.7962415469915555, + "pass@16": 0.8375704897171268, + "pass@32": 0.8683757778577055, + "pass@64": 0.8930159138899741, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.3810763888888889, + "pass@2": 0.4868089906229668, + "pass@4": 0.5760386284513578, + "pass@8": 0.6432655475632902, + "pass@16": 0.6953101222141723, + "pass@32": 0.7400005671118122, + "pass@64": 0.7801792827865532, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.29670516304347827, + "pass@2": 0.3995455476353497, + "pass@4": 0.5012611435022484, + "pass@8": 0.590376899556164, + "pass@16": 0.6594289276417519, + "pass@32": 0.711161429942676, + "pass@64": 0.7534745261944554, + "pass@128": 0.7857142857142857 + }, + "18": { + "pass@1": 0.23887867647058825, + "pass@2": 0.3285324513663731, + "pass@4": 0.4181968639030413, + "pass@8": 0.5009363789683653, + "pass@16": 0.5731285778604139, + "pass@32": 0.6325426409355972, + "pass@64": 0.6800289019744267, + "pass@128": 0.7205882352941176 + }, + "19": { + "pass@1": 0.17503720238095238, + "pass@2": 0.24297329044806892, + "pass@4": 0.31580475850116946, + "pass@8": 0.3867082039757981, + "pass@16": 0.44867577405752523, + "pass@32": 0.5014357515012002, + "pass@64": 0.5501519038964435, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.1511508233532934, + "pass@2": 0.217524842048187, + "pass@4": 0.28555462752784627, + "pass@8": 0.3501604003392409, + "pass@16": 0.4087684917529828, + "pass@32": 0.4609191551164828, + "pass@64": 0.5082929397545116, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587333, + "answer_accuracy": 0.7331105707780796, + "avg_response_len": 292.55482230587955, + "resp_tokens_sum": 234380881, + "pass_at_k": { + "pass@1": 0.7331105707780796, + "pass@2": 0.7719972609521001, + "pass@4": 0.8051676618290551, + "pass@8": 0.8328356007953099, + "pass@16": 0.8557784222570307, + "pass@32": 0.8745643633290906, + "pass@64": 0.8897046142733968, + "pass@128": 0.9027001118389519 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9627332089552239, + "pass@2": 0.9724475555294394, + "pass@4": 0.9788069581787349, + "pass@8": 0.9837743366957562, + "pass@16": 0.9883754773938696, + "pass@32": 0.991781623197491, + "pass@64": 0.9935765469707158, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961397058823529, + "pass@2": 0.9988312007874015, + "pass@4": 0.9998461672070401, + "pass@8": 0.9999959070953863, + "pass@16": 0.9999999974796792, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996549684542587, + "pass@2": 0.9999646818102784, + "pass@4": 0.9999997039999338, + "pass@8": 0.9999999999933739, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993384009009009, + "pass@2": 0.9963785143411128, + "pass@4": 0.9985127714891495, + "pass@8": 0.9995892765946597, + "pass@16": 0.9999566515257909, + "pass@32": 0.9999997093889322, + "pass@64": 0.9999999999997521, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962067926045016, + "pass@2": 0.9980144852773629, + "pass@4": 0.999233284746159, + "pass@8": 0.9998544254815018, + "pass@16": 0.9999950128136015, + "pass@32": 0.9999999978428284, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.944766865079365, + "pass@2": 0.9588774059492565, + "pass@4": 0.9696854297974654, + "pass@8": 0.9779315603426709, + "pass@16": 0.9829546879178315, + "pass@32": 0.9856370532004864, + "pass@64": 0.9869108488629128, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.890266628440367, + "pass@2": 0.9240090509041881, + "pass@4": 0.9420546302125073, + "pass@8": 0.9536679236154701, + "pass@16": 0.9622388168637205, + "pass@32": 0.9693577252308863, + "pass@64": 0.9751121135669748, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6949990354938271, + "pass@2": 0.7612535846213664, + "pass@4": 0.8086492296101871, + "pass@8": 0.8472164652841165, + "pass@16": 0.8797007013342069, + "pass@32": 0.9054289462435336, + "pass@64": 0.9247574085886842, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.648321587537092, + "pass@2": 0.7306515631206338, + "pass@4": 0.7862802549347504, + "pass@8": 0.8249572423306274, + "pass@16": 0.8559579355913478, + "pass@32": 0.883398091383188, + "pass@64": 0.9059038743502413, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5392071759259259, + "pass@2": 0.6344817791873233, + "pass@4": 0.7079802536257039, + "pass@8": 0.7642526753150662, + "pass@16": 0.8071152718012273, + "pass@32": 0.8398910919721222, + "pass@64": 0.865244693493098, + "pass@128": 0.8858024691358025 + }, + "16": { + "pass@1": 0.37317938790560473, + "pass@2": 0.47691762188465375, + "pass@4": 0.5665855180713028, + "pass@8": 0.6412671657376158, + "pass@16": 0.7045272977137541, + "pass@32": 0.7566904327098634, + "pass@64": 0.7969800612318285, + "pass@128": 0.831858407079646 + }, + "17": { + "pass@1": 0.2708019578313253, + "pass@2": 0.36564775341523553, + "pass@4": 0.45550395959541135, + "pass@8": 0.5304451165455887, + "pass@16": 0.5906134572060683, + "pass@32": 0.6409475965285415, + "pass@64": 0.6825230999818859, + "pass@128": 0.7168674698795181 + }, + "18": { + "pass@1": 0.25491431451612906, + "pass@2": 0.33559817119634233, + "pass@4": 0.4112166173784727, + "pass@8": 0.478386963179724, + "pass@16": 0.5365025710767365, + "pass@32": 0.5853870688476496, + "pass@64": 0.6257388038820341, + "pass@128": 0.6612903225806451 + }, + "19": { + "pass@1": 0.1848800981873112, + "pass@2": 0.26643789994528644, + "pass@4": 0.3507837253915768, + "pass@8": 0.4257836524048795, + "pass@16": 0.4885780364032442, + "pass@32": 0.540870567295929, + "pass@64": 0.5872913420960638, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1655170795795796, + "pass@2": 0.2355894280106878, + "pass@4": 0.31134118398488475, + "pass@8": 0.38612888463663003, + "pass@16": 0.4534678038864786, + "pass@32": 0.5080969934218494, + "pass@64": 0.5515849815450634, + "pass@128": 0.5915915915915916 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f4c992dfb93390b85cae0b878a143b325c6ad66d --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e261a43540d36659c6ba12dff93cb148ce7ee709c86822b6b0c8bc20485796 +size 1789483125 diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..785865028e54bcf3eba448fd9de12ba152097265 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9942239583333333, + "avg_response_len": 200.42143663194443, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9942239583333333, + "pass@2": 0.9962396106736653, + "pass@4": 0.9976002042973796, + "pass@8": 0.9984843356137656, + "pass@16": 0.999020183855886, + "pass@32": 0.9993363892976035, + "pass@64": 0.9995430390655977, + "pass@128": 0.9996666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9643125, + "pass@2": 0.9755124261811027, + "pass@4": 0.9839911730408695, + "pass@8": 0.9895877994173273, + "pass@16": 0.9929323831323262, + "pass@32": 0.9951233508694214, + "pass@64": 0.9968878851582194, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9993046875, + "pass@2": 0.9998324311023622, + "pass@4": 0.9999897678102737, + "pass@8": 0.9999999660823697, + "pass@16": 0.9999999999998981, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961484375, + "pass@2": 0.9978601131889766, + "pass@4": 0.9987904610048742, + "pass@8": 0.9993821593513503, + "pass@16": 0.9997897536795602, + "pass@32": 0.9999733721668991, + "pass@64": 0.9999998615760638, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993671875, + "pass@2": 0.9998283710629923, + "pass@4": 0.9999781246719159, + "pass@8": 0.9999996244516949, + "pass@16": 0.9999999999472515, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983359375, + "pass@2": 0.9994175688976378, + "pass@4": 0.9998698080239969, + "pass@8": 0.9999891898079931, + "pass@16": 0.9999999279186353, + "pass@32": 0.9999999999993945, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9943984375, + "pass@2": 0.9963143454724409, + "pass@4": 0.9977673551743532, + "pass@8": 0.9989378554425637, + "pass@16": 0.9996780348505084, + "pass@32": 0.9999670234086364, + "pass@64": 0.9999999104299374, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961484375, + "pass@2": 0.9973912401574803, + "pass@4": 0.9980151489501311, + "pass@8": 0.9984624259705912, + "pass@16": 0.9987815551747922, + "pass@32": 0.9989637572340796, + "pass@64": 0.9989996944261581, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9643125, + "2": 1.0, + "3": 1.0, + "4": 0.9993046875, + "5": 0.9961484375, + "6": 0.9993671875, + "7": 0.9983359375, + "8": 0.9943984375, + "9": 0.9961484375 + }, + "per_op_avg_response_len": { + "10": 281.8471640625, + "2": 115.7379140625, + "3": 144.9791328125, + "4": 160.1477265625, + "5": 173.2182265625, + "6": 199.351765625, + "7": 224.4176953125, + "8": 244.1198984375, + "9": 259.97340625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.8471640625, + "2": 115.7379140625, + "3": 144.9791328125, + "4": 160.1477265625, + "5": 173.2182265625, + "6": 199.351765625, + "7": 224.4176953125, + "8": 244.1198984375, + "9": 259.97340625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377047, + "answer_accuracy": 0.9948259667342114, + "avg_response_len": 190.94212523218508, + "resp_tokens_sum": 72368593, + "pass_at_k": { + "pass@1": 0.9948259667342114, + "pass@2": 0.9967543119610047, + "pass@4": 0.9978659654625239, + "pass@8": 0.9985604741319131, + "pass@16": 0.9990599862792882, + "pass@32": 0.9993957534287397, + "pass@64": 0.9996008492318896, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9670907079646017, + "pass@2": 0.9783562556616267, + "pass@4": 0.9863912442360632, + "pass@8": 0.9916879946510442, + "pass@16": 0.9952884427643433, + "pass@32": 0.9977497176863996, + "pass@64": 0.9994638731503346, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9978271484375, + "pass@2": 0.9994763471948819, + "pass@4": 0.9999680244071053, + "pass@8": 0.9999998940074054, + "pass@16": 0.9999999999996815, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9959325396825397, + "pass@2": 0.9971773059617548, + "pass@4": 0.9978584415043358, + "pass@8": 0.9985893161342536, + "pass@16": 0.9994090073019657, + "pass@32": 0.9999164177993916, + "pass@64": 0.9999995605640043, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 0.9999963164222737, + "pass@4": 0.9999999985967324, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9980815458579881, + "pass@2": 0.9994394422960443, + "pass@4": 0.9999394495066816, + "pass@8": 0.9999992102443163, + "pass@16": 0.9999999999285286, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999198717948718, + "pass@2": 0.9999499198717948, + "pass@4": 0.9999994531933509, + "pass@8": 0.9999999999711477, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9964788732394366, + "pass@2": 0.9971543334812022, + "pass@4": 0.9971829639252839, + "pass@8": 0.997183098590574, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388560, + "answer_accuracy": 0.9946346657929227, + "avg_response_len": 198.3889867300131, + "resp_tokens_sum": 77501848, + "pass_at_k": { + "pass@1": 0.9946346657929227, + "pass@2": 0.9966231240518675, + "pass@4": 0.9979855741328536, + "pass@8": 0.9987879532310419, + "pass@16": 0.999182052851467, + "pass@32": 0.999456233913557, + "pass@64": 0.9997352895963734, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9608656058282209, + "pass@2": 0.9739629878991349, + "pass@4": 0.9843521879550335, + "pass@8": 0.9903643149049658, + "pass@16": 0.9930190213003309, + "pass@32": 0.9950204756095313, + "pass@64": 0.997522728288262, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.998935688405797, + "pass@2": 0.9998541452698849, + "pass@4": 0.9999957437385544, + "pass@8": 0.9999999968300656, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994627507163324, + "pass@2": 0.9999439484015071, + "pass@4": 0.999999177842741, + "pass@8": 0.9999999999123027, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994806719653179, + "pass@2": 0.9999505740521598, + "pass@4": 0.9999993552106564, + "pass@8": 0.9999999999508568, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995400528169014, + "pass@2": 0.997621506598647, + "pass@4": 0.9990206005059226, + "pass@8": 0.999771949976716, + "pass@16": 0.9999861357166676, + "pass@32": 0.9999999749699509, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962574850299402, + "pass@2": 0.9973448771747843, + "pass@4": 0.9979126474273051, + "pass@8": 0.9985719326395935, + "pass@16": 0.9993543298783014, + "pass@32": 0.9998914962309466, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379739, + "answer_accuracy": 0.9932075451958486, + "avg_response_len": 211.89491442082357, + "resp_tokens_sum": 81015054, + "pass_at_k": { + "pass@1": 0.9932075451958486, + "pass@2": 0.9953375305457507, + "pass@4": 0.9969430005652538, + "pass@8": 0.9980986352052746, + "pass@16": 0.9988153364671316, + "pass@32": 0.9991550893444113, + "pass@64": 0.9992892980135997, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648554104477612, + "pass@2": 0.9741424521095311, + "pass@4": 0.9812111282731446, + "pass@8": 0.9867068733480728, + "pass@16": 0.9904638808695707, + "pass@32": 0.9925657358955954, + "pass@64": 0.993663351457559, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935202205882353, + "pass@2": 0.9964693579203334, + "pass@4": 0.9984309421800216, + "pass@8": 0.9994897819490819, + "pass@16": 0.9999291658218862, + "pass@32": 0.9999991192943843, + "pass@64": 0.9999999999953015, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987184542586751, + "pass@2": 0.9995241747186965, + "pass@4": 0.9999318992854599, + "pass@8": 0.9999988154015811, + "pass@16": 0.9999999998336015, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997354628164557, + "pass@2": 0.9988105657579986, + "pass@4": 0.9996534721134541, + "pass@8": 0.9999666353241072, + "pass@16": 0.9999997719708628, + "pass@32": 0.9999999999980841, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9888325825825826, + "pass@2": 0.9915145214111751, + "pass@4": 0.9943399777730485, + "pass@8": 0.9970534931226165, + "pass@16": 0.9990479179312053, + "pass@32": 0.9999009978807923, + "pass@64": 0.9999997310208336, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956541398713826, + "pass@2": 0.997711455933362, + "pass@4": 0.9990751527843585, + "pass@8": 0.9998051462035797, + "pass@16": 0.9999910257088084, + "pass@32": 0.9999999919387248, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5867765625, + "avg_response_len": 358.94868359375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5867765625, + "pass@2": 0.6586586122047158, + "pass@4": 0.7184700836520341, + "pass@8": 0.7663182013135811, + "pass@16": 0.8033292509069808, + "pass@32": 0.8321137031810789, + "pass@64": 0.8553770726284838, + "pass@128": 0.8758 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.960546875, + "pass@2": 0.9706694143700787, + "pass@4": 0.9774134248218973, + "pass@8": 0.9819330569973834, + "pass@16": 0.9849754009158007, + "pass@32": 0.9873440048532779, + "pass@64": 0.9893877747959311, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9120546875, + "pass@2": 0.9313029035433076, + "pass@4": 0.9433419732845894, + "pass@8": 0.9522147605754815, + "pass@16": 0.9594768598247163, + "pass@32": 0.9660329347944124, + "pass@64": 0.9718429809322134, + "pass@128": 0.977 + }, + "13": { + "pass@1": 0.824078125, + "pass@2": 0.8683841043307097, + "pass@4": 0.8969351185789278, + "pass@8": 0.9158503351499402, + "pass@16": 0.9295555575072398, + "pass@32": 0.9411213923443775, + "pass@64": 0.9505974589389412, + "pass@128": 0.959 + }, + "14": { + "pass@1": 0.8063046875, + "pass@2": 0.8646321358267728, + "pass@4": 0.8984279219160112, + "pass@8": 0.9185222175704016, + "pass@16": 0.9311341631703898, + "pass@32": 0.9401766943307132, + "pass@64": 0.9481055559462812, + "pass@128": 0.956 + }, + "15": { + "pass@1": 0.70509375, + "pass@2": 0.7868759842519693, + "pass@4": 0.8402833769216345, + "pass@8": 0.8759016943178962, + "pass@16": 0.9005518621162062, + "pass@32": 0.9187405822278949, + "pass@64": 0.9322985098563336, + "pass@128": 0.943 + }, + "16": { + "pass@1": 0.5166875, + "pass@2": 0.6321081446850385, + "pass@4": 0.720487092894637, + "pass@8": 0.7840962525217886, + "pass@16": 0.829594288594719, + "pass@32": 0.8629783855643571, + "pass@64": 0.8873063287950224, + "pass@128": 0.906 + }, + "17": { + "pass@1": 0.4163359375, + "pass@2": 0.5318865649606304, + "pass@4": 0.629992870359955, + "pass@8": 0.7052914180891698, + "pass@16": 0.7588230188351526, + "pass@32": 0.7968780074093672, + "pass@64": 0.8265044316885863, + "pass@128": 0.853 + }, + "18": { + "pass@1": 0.3228046875, + "pass@2": 0.4227716535433074, + "pass@4": 0.5146231126734155, + "pass@8": 0.5934185734745823, + "pass@16": 0.6578459420169069, + "pass@32": 0.7100692194245987, + "pass@64": 0.7538790497335078, + "pass@128": 0.793 + }, + "19": { + "pass@1": 0.2160390625, + "pass@2": 0.3075067667322835, + "pass@4": 0.40434608098987623, + "pass@8": 0.4936801501092722, + "pass@16": 0.5674934737312514, + "pass@32": 0.6267076842610163, + "pass@64": 0.676281981088532, + "pass@128": 0.721 + }, + "20": { + "pass@1": 0.1878203125, + "pass@2": 0.2704484498031496, + "pass@4": 0.3588498640794897, + "pass@8": 0.4422735543299612, + "pass@16": 0.5138419423573215, + "pass@32": 0.5710881266007416, + "pass@64": 0.6175666545094013, + "pass@128": 0.659 + } + }, + "per_op_accuracy": { + "11": 0.960546875, + "12": 0.9120546875, + "13": 0.824078125, + "14": 0.8063046875, + "15": 0.70509375, + "16": 0.5166875, + "17": 0.4163359375, + "18": 0.3228046875, + "19": 0.2160390625, + "20": 0.1878203125 + }, + "per_op_avg_response_len": { + "11": 299.6658515625, + "12": 310.2260390625, + "13": 329.6556015625, + "14": 354.9960859375, + "15": 369.342890625, + "16": 373.52596875, + "17": 386.6290078125, + "18": 387.452953125, + "19": 388.1441875, + "20": 389.84825 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.6658515625, + "12": 310.2260390625, + "13": 329.6556015625, + "14": 354.9960859375, + "15": 369.342890625, + "16": 373.52596875, + "17": 386.6290078125, + "18": 387.452953125, + "19": 388.1441875, + "20": 389.84825 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 253554, + "answer_accuracy": 0.5886747771173848, + "avg_response_len": 341.30865063150077, + "resp_tokens_sum": 147008462, + "pass_at_k": { + "pass@1": 0.5886747771173848, + "pass@2": 0.659568486679698, + "pass@4": 0.7183372159646492, + "pass@8": 0.7651668018473232, + "pass@16": 0.8012637796808173, + "pass@32": 0.829617942753822, + "pass@64": 0.8528915979764461, + "pass@128": 0.8736998514115899 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9505577195467422, + "pass@2": 0.9641180210122462, + "pass@4": 0.9734464953566355, + "pass@8": 0.9799640965041405, + "pass@16": 0.9847543053720864, + "pass@32": 0.9886326398519015, + "pass@64": 0.9918760253150202, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.9008430437853108, + "pass@2": 0.9205685723341787, + "pass@4": 0.9321145698948646, + "pass@8": 0.9403918855715436, + "pass@16": 0.9476097691840198, + "pass@32": 0.955394921972066, + "pass@64": 0.963951534680491, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8344435736677116, + "pass@2": 0.8757532304445487, + "pass@4": 0.9049746909575169, + "pass@8": 0.92617544555909, + "pass@16": 0.9417792383220905, + "pass@32": 0.9544322640829537, + "pass@64": 0.9637782892800076, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8111127336448598, + "pass@2": 0.8645607200677018, + "pass@4": 0.8928378710254671, + "pass@8": 0.909400920399937, + "pass@16": 0.922035389489197, + "pass@32": 0.9330553564330485, + "pass@64": 0.9418133713811047, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.7193370664739884, + "pass@2": 0.7990888557644162, + "pass@4": 0.8498221508438609, + "pass@8": 0.883376836515064, + "pass@16": 0.9057129592960427, + "pass@32": 0.9217041139833264, + "pass@64": 0.9337069258447572, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5192288306451613, + "pass@2": 0.6311301435102868, + "pass@4": 0.7141203666073994, + "pass@8": 0.770769853314818, + "pass@16": 0.8121276516366241, + "pass@32": 0.8450246042301767, + "pass@64": 0.8711062163496095, + "pass@128": 0.8935483870967742 + }, + "17": { + "pass@1": 0.4158688583815029, + "pass@2": 0.5342351139001411, + "pass@4": 0.6372437631531604, + "pass@8": 0.7165662520336787, + "pass@16": 0.7695623182836836, + "pass@32": 0.8039757451384392, + "pass@64": 0.8285737983798428, + "pass@128": 0.8497109826589595 + }, + "18": { + "pass@1": 0.30189732142857145, + "pass@2": 0.4026367407199098, + "pass@4": 0.49605475601264115, + "pass@8": 0.5761246431366777, + "pass@16": 0.6422153424913305, + "pass@32": 0.6975131016174686, + "pass@64": 0.745362531715204, + "pass@128": 0.7885714285714286 + }, + "19": { + "pass@1": 0.21506662912912913, + "pass@2": 0.304116862925918, + "pass@4": 0.3969523421058855, + "pass@8": 0.48389617153847675, + "pass@16": 0.5573617470631065, + "pass@32": 0.6160098154126928, + "pass@64": 0.6654528414928708, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20680837087087087, + "pass@2": 0.29180041261734174, + "pass@4": 0.3802260170744425, + "pass@8": 0.4603380140858725, + "pass@16": 0.5255192306118455, + "pass@32": 0.5771437801720317, + "pass@64": 0.6203514730224098, + "pass@128": 0.6606606606606606 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 255794, + "answer_accuracy": 0.5942285533749628, + "avg_response_len": 354.9730267804044, + "resp_tokens_sum": 152803109, + "pass_at_k": { + "pass@1": 0.5942285533749628, + "pass@2": 0.666404579947137, + "pass@4": 0.7264719183462933, + "pass@8": 0.7747569522750786, + "pass@16": 0.812536581590751, + "pass@32": 0.8420002813302753, + "pass@64": 0.8656503021483124, + "pass@128": 0.8867082961641392 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9690323795180723, + "pass@2": 0.975195812778674, + "pass@4": 0.9779912812103307, + "pass@8": 0.9793962882487052, + "pass@16": 0.9802826047105917, + "pass@32": 0.9814200443385674, + "pass@64": 0.9830661227587516, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9060295846394985, + "pass@2": 0.9250572347147835, + "pass@4": 0.9364256305736081, + "pass@8": 0.9444015269184481, + "pass@16": 0.9506101312135917, + "pass@32": 0.9556181429280108, + "pass@64": 0.9604398982497184, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8360031512605042, + "pass@2": 0.8767200285626062, + "pass@4": 0.9016726128771714, + "pass@8": 0.9169013117277433, + "pass@16": 0.9277848626738392, + "pass@32": 0.937749843120475, + "pass@64": 0.9470916007788368, + "pass@128": 0.957983193277311 + }, + "14": { + "pass@1": 0.8244243421052632, + "pass@2": 0.8814703055210201, + "pass@4": 0.9161977258325157, + "pass@8": 0.9387559253266154, + "pass@16": 0.9528099888394235, + "pass@32": 0.9609495059700786, + "pass@64": 0.9663334643731861, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.718655303030303, + "pass@2": 0.8018458452636601, + "pass@4": 0.8560838676415438, + "pass@8": 0.8931015580401608, + "pass@16": 0.9194444646380041, + "pass@32": 0.9378258206854141, + "pass@64": 0.9487053603010248, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5079015313390314, + "pass@2": 0.6259958190322363, + "pass@4": 0.7192196907544668, + "pass@8": 0.7885741943570277, + "pass@16": 0.8390539563285804, + "pass@32": 0.8758487348681383, + "pass@64": 0.903097786845089, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.44429347826086957, + "pass@2": 0.5645040378784174, + "pass@4": 0.6619254227538325, + "pass@8": 0.7319004490853245, + "pass@16": 0.7814428010063853, + "pass@32": 0.8193750801545913, + "pass@64": 0.8522784760506044, + "pass@128": 0.8819875776397516 + }, + "18": { + "pass@1": 0.3361443014705882, + "pass@2": 0.43848533754052793, + "pass@4": 0.5339143221067955, + "pass@8": 0.6180126229671818, + "pass@16": 0.6884203893465171, + "pass@32": 0.7449961226518379, + "pass@64": 0.7898113276493597, + "pass@128": 0.8294117647058824 + }, + "19": { + "pass@1": 0.22470238095238096, + "pass@2": 0.3165102409073866, + "pass@4": 0.41335825823111383, + "pass@8": 0.5032275614596812, + "pass@16": 0.5792477084209239, + "pass@32": 0.6428269554995669, + "pass@64": 0.6971000675830643, + "pass@128": 0.7470238095238095 + }, + "20": { + "pass@1": 0.17652975299401197, + "pass@2": 0.25829946897543493, + "pass@4": 0.34719286427145707, + "pass@8": 0.4320416991906228, + "pass@16": 0.5046182598137827, + "pass@32": 0.5614345432315808, + "pass@64": 0.6064991139359319, + "pass@128": 0.6497005988023952 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 241726, + "answer_accuracy": 0.5771651512836186, + "avg_response_len": 381.1763256418093, + "resp_tokens_sum": 159642744, + "pass_at_k": { + "pass@1": 0.5771651512836186, + "pass@2": 0.6497614798312602, + "pass@4": 0.7103823482276385, + "pass@8": 0.7588288797122865, + "pass@16": 0.7959900264529708, + "pass@32": 0.8245188595141839, + "pass@64": 0.8473742460174775, + "pass@128": 0.8667481662591687 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9627976190476191, + "pass@2": 0.9732404543182105, + "pass@4": 0.9812498622196033, + "pass@8": 0.986813216612229, + "pass@16": 0.9901692265255165, + "pass@32": 0.9921435817941344, + "pass@64": 0.9932621749327728, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9300697629969419, + "pass@2": 0.949016425269089, + "pass@4": 0.9622435455315791, + "pass@8": 0.9726359204775846, + "pass@16": 0.9809736075732047, + "pass@32": 0.9877093113830763, + "pass@64": 0.9915101226717405, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.800733024691358, + "pass@2": 0.851943745747059, + "pass@4": 0.8837995968096578, + "pass@8": 0.9045265423758843, + "pass@16": 0.9194715571232489, + "pass@32": 0.931730883049093, + "pass@64": 0.94148297277955, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7833364243323442, + "pass@2": 0.8476121667562327, + "pass@4": 0.8857191189380257, + "pass@8": 0.9066765449861103, + "pass@16": 0.9178035221402799, + "pass@32": 0.9258789135725163, + "pass@64": 0.9356006496062804, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6760706018518519, + "pass@2": 0.7585867630261495, + "pass@4": 0.8140038160739167, + "pass@8": 0.8504006010198468, + "pass@16": 0.8757978545346746, + "pass@32": 0.8961371542082629, + "pass@64": 0.9140837796750277, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.5234605457227138, + "pass@2": 0.6393311732051187, + "pass@4": 0.7276214389130556, + "pass@8": 0.7916461822267785, + "pass@16": 0.8357722062419856, + "pass@32": 0.866070360809102, + "pass@64": 0.8857701431976893, + "pass@128": 0.8997050147492626 + }, + "17": { + "pass@1": 0.3897072665662651, + "pass@2": 0.4978039619106348, + "pass@4": 0.5914654885006841, + "pass@8": 0.6677335851808504, + "pass@16": 0.7256923939305502, + "pass@32": 0.7676615174147252, + "pass@64": 0.7993500848218832, + "pass@128": 0.8283132530120482 + }, + "18": { + "pass@1": 0.33177923387096775, + "pass@2": 0.4282702565405131, + "pass@4": 0.5144292856538094, + "pass@8": 0.5859698598964601, + "pass@16": 0.6419601282810513, + "pass@32": 0.6859385553447067, + "pass@64": 0.7240848781690444, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.20822318731117825, + "pass@2": 0.30177767502438324, + "pass@4": 0.40263618218417563, + "pass@8": 0.493831614309686, + "pass@16": 0.5657546281867276, + "pass@32": 0.6211074280384756, + "pass@64": 0.6660439944516493, + "pass@128": 0.7039274924471299 + }, + "20": { + "pass@1": 0.18015671921921922, + "pass@2": 0.2612819512425812, + "pass@4": 0.34916571689800047, + "pass@8": 0.43447167600509806, + "pass@16": 0.5114160353927147, + "pass@32": 0.5747150461384618, + "pass@64": 0.625882612427439, + "pass@128": 0.6666666666666666 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7797779605263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7797779605263158, + "pass@2": 0.8185654009531573, + "pass@4": 0.850689614484026, + "pass@8": 0.8762916333505202, + "pass@16": 0.8960249559880326, + "pass@32": 0.9113244492362866, + "pass@64": 0.9236662146250088, + "pass@128": 0.9344736842105263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9643125, + "pass@2": 0.9755124261811027, + "pass@4": 0.9839911730408695, + "pass@8": 0.9895877994173273, + "pass@16": 0.9929323831323262, + "pass@32": 0.9951233508694214, + "pass@64": 0.9968878851582194, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9993046875, + "pass@2": 0.9998324311023622, + "pass@4": 0.9999897678102737, + "pass@8": 0.9999999660823697, + "pass@16": 0.9999999999998981, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961484375, + "pass@2": 0.9978601131889766, + "pass@4": 0.9987904610048742, + "pass@8": 0.9993821593513503, + "pass@16": 0.9997897536795602, + "pass@32": 0.9999733721668991, + "pass@64": 0.9999998615760638, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993671875, + "pass@2": 0.9998283710629923, + "pass@4": 0.9999781246719159, + "pass@8": 0.9999996244516949, + "pass@16": 0.9999999999472515, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983359375, + "pass@2": 0.9994175688976378, + "pass@4": 0.9998698080239969, + "pass@8": 0.9999891898079931, + "pass@16": 0.9999999279186353, + "pass@32": 0.9999999999993945, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9943984375, + "pass@2": 0.9963143454724409, + "pass@4": 0.9977673551743532, + "pass@8": 0.9989378554425637, + "pass@16": 0.9996780348505084, + "pass@32": 0.9999670234086364, + "pass@64": 0.9999999104299374, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961484375, + "pass@2": 0.9973912401574803, + "pass@4": 0.9980151489501311, + "pass@8": 0.9984624259705912, + "pass@16": 0.9987815551747922, + "pass@32": 0.9989637572340796, + "pass@64": 0.9989996944261581, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.960546875, + "pass@2": 0.9706694143700787, + "pass@4": 0.9774134248218973, + "pass@8": 0.9819330569973834, + "pass@16": 0.9849754009158007, + "pass@32": 0.9873440048532779, + "pass@64": 0.9893877747959311, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9120546875, + "pass@2": 0.9313029035433076, + "pass@4": 0.9433419732845894, + "pass@8": 0.9522147605754815, + "pass@16": 0.9594768598247163, + "pass@32": 0.9660329347944124, + "pass@64": 0.9718429809322134, + "pass@128": 0.977 + }, + "13": { + "pass@1": 0.824078125, + "pass@2": 0.8683841043307097, + "pass@4": 0.8969351185789278, + "pass@8": 0.9158503351499402, + "pass@16": 0.9295555575072398, + "pass@32": 0.9411213923443775, + "pass@64": 0.9505974589389412, + "pass@128": 0.959 + }, + "14": { + "pass@1": 0.8063046875, + "pass@2": 0.8646321358267728, + "pass@4": 0.8984279219160112, + "pass@8": 0.9185222175704016, + "pass@16": 0.9311341631703898, + "pass@32": 0.9401766943307132, + "pass@64": 0.9481055559462812, + "pass@128": 0.956 + }, + "15": { + "pass@1": 0.70509375, + "pass@2": 0.7868759842519693, + "pass@4": 0.8402833769216345, + "pass@8": 0.8759016943178962, + "pass@16": 0.9005518621162062, + "pass@32": 0.9187405822278949, + "pass@64": 0.9322985098563336, + "pass@128": 0.943 + }, + "16": { + "pass@1": 0.5166875, + "pass@2": 0.6321081446850385, + "pass@4": 0.720487092894637, + "pass@8": 0.7840962525217886, + "pass@16": 0.829594288594719, + "pass@32": 0.8629783855643571, + "pass@64": 0.8873063287950224, + "pass@128": 0.906 + }, + "17": { + "pass@1": 0.4163359375, + "pass@2": 0.5318865649606304, + "pass@4": 0.629992870359955, + "pass@8": 0.7052914180891698, + "pass@16": 0.7588230188351526, + "pass@32": 0.7968780074093672, + "pass@64": 0.8265044316885863, + "pass@128": 0.853 + }, + "18": { + "pass@1": 0.3228046875, + "pass@2": 0.4227716535433074, + "pass@4": 0.5146231126734155, + "pass@8": 0.5934185734745823, + "pass@16": 0.6578459420169069, + "pass@32": 0.7100692194245987, + "pass@64": 0.7538790497335078, + "pass@128": 0.793 + }, + "19": { + "pass@1": 0.2160390625, + "pass@2": 0.3075067667322835, + "pass@4": 0.40434608098987623, + "pass@8": 0.4936801501092722, + "pass@16": 0.5674934737312514, + "pass@32": 0.6267076842610163, + "pass@64": 0.676281981088532, + "pass@128": 0.721 + }, + "20": { + "pass@1": 0.1878203125, + "pass@2": 0.2704484498031496, + "pass@4": 0.3588498640794897, + "pass@8": 0.4422735543299612, + "pass@16": 0.5138419423573215, + "pass@32": 0.5710881266007416, + "pass@64": 0.6175666545094013, + "pass@128": 0.659 + } + }, + "per_op_accuracy": { + "10": 0.9643125, + "2": 1.0, + "3": 1.0, + "4": 0.9993046875, + "5": 0.9961484375, + "6": 0.9993671875, + "7": 0.9983359375, + "8": 0.9943984375, + "9": 0.9961484375, + "11": 0.960546875, + "12": 0.9120546875, + "13": 0.824078125, + "14": 0.8063046875, + "15": 0.70509375, + "16": 0.5166875, + "17": 0.4163359375, + "18": 0.3228046875, + "19": 0.2160390625, + "20": 0.1878203125 + }, + "per_op_avg_response_len": { + "10": 281.8471640625, + "2": 115.7379140625, + "3": 144.9791328125, + "4": 160.1477265625, + "5": 173.2182265625, + "6": 199.351765625, + "7": 224.4176953125, + "8": 244.1198984375, + "9": 259.97340625, + "11": 299.6658515625, + "12": 310.2260390625, + "13": 329.6556015625, + "14": 354.9960859375, + "15": 369.342890625, + "16": 373.52596875, + "17": 386.6290078125, + "18": 387.452953125, + "19": 388.1441875, + "20": 389.84825 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.8471640625, + "2": 115.7379140625, + "3": 144.9791328125, + "4": 160.1477265625, + "5": 173.2182265625, + "6": 199.351765625, + "7": 224.4176953125, + "8": 244.1198984375, + "9": 259.97340625, + "11": 299.6658515625, + "12": 310.2260390625, + "13": 329.6556015625, + "14": 354.9960859375, + "15": 369.342890625, + "16": 373.52596875, + "17": 386.6290078125, + "18": 387.452953125, + "19": 388.1441875, + "20": 389.84825 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 630601, + "answer_accuracy": 0.7787812697597217, + "avg_response_len": 270.9268482749763, + "resp_tokens_sum": 219377055, + "pass_at_k": { + "pass@1": 0.7787812697597217, + "pass@2": 0.8173944791959636, + "pass@4": 0.8491757596357156, + "pass@8": 0.8744109788366741, + "pass@16": 0.8938459117924357, + "pass@32": 0.9090855522082075, + "pass@64": 0.9215615462798628, + "pass@128": 0.9326588681631363 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9670907079646017, + "pass@2": 0.9783562556616267, + "pass@4": 0.9863912442360632, + "pass@8": 0.9916879946510442, + "pass@16": 0.9952884427643433, + "pass@32": 0.9977497176863996, + "pass@64": 0.9994638731503346, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9978271484375, + "pass@2": 0.9994763471948819, + "pass@4": 0.9999680244071053, + "pass@8": 0.9999998940074054, + "pass@16": 0.9999999999996815, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9959325396825397, + "pass@2": 0.9971773059617548, + "pass@4": 0.9978584415043358, + "pass@8": 0.9985893161342536, + "pass@16": 0.9994090073019657, + "pass@32": 0.9999164177993916, + "pass@64": 0.9999995605640043, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 0.9999963164222737, + "pass@4": 0.9999999985967324, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9980815458579881, + "pass@2": 0.9994394422960443, + "pass@4": 0.9999394495066816, + "pass@8": 0.9999992102443163, + "pass@16": 0.9999999999285286, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999198717948718, + "pass@2": 0.9999499198717948, + "pass@4": 0.9999994531933509, + "pass@8": 0.9999999999711477, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9964788732394366, + "pass@2": 0.9971543334812022, + "pass@4": 0.9971829639252839, + "pass@8": 0.997183098590574, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9505577195467422, + "pass@2": 0.9641180210122462, + "pass@4": 0.9734464953566355, + "pass@8": 0.9799640965041405, + "pass@16": 0.9847543053720864, + "pass@32": 0.9886326398519015, + "pass@64": 0.9918760253150202, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.9008430437853108, + "pass@2": 0.9205685723341787, + "pass@4": 0.9321145698948646, + "pass@8": 0.9403918855715436, + "pass@16": 0.9476097691840198, + "pass@32": 0.955394921972066, + "pass@64": 0.963951534680491, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8344435736677116, + "pass@2": 0.8757532304445487, + "pass@4": 0.9049746909575169, + "pass@8": 0.92617544555909, + "pass@16": 0.9417792383220905, + "pass@32": 0.9544322640829537, + "pass@64": 0.9637782892800076, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8111127336448598, + "pass@2": 0.8645607200677018, + "pass@4": 0.8928378710254671, + "pass@8": 0.909400920399937, + "pass@16": 0.922035389489197, + "pass@32": 0.9330553564330485, + "pass@64": 0.9418133713811047, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.7193370664739884, + "pass@2": 0.7990888557644162, + "pass@4": 0.8498221508438609, + "pass@8": 0.883376836515064, + "pass@16": 0.9057129592960427, + "pass@32": 0.9217041139833264, + "pass@64": 0.9337069258447572, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5192288306451613, + "pass@2": 0.6311301435102868, + "pass@4": 0.7141203666073994, + "pass@8": 0.770769853314818, + "pass@16": 0.8121276516366241, + "pass@32": 0.8450246042301767, + "pass@64": 0.8711062163496095, + "pass@128": 0.8935483870967742 + }, + "17": { + "pass@1": 0.4158688583815029, + "pass@2": 0.5342351139001411, + "pass@4": 0.6372437631531604, + "pass@8": 0.7165662520336787, + "pass@16": 0.7695623182836836, + "pass@32": 0.8039757451384392, + "pass@64": 0.8285737983798428, + "pass@128": 0.8497109826589595 + }, + "18": { + "pass@1": 0.30189732142857145, + "pass@2": 0.4026367407199098, + "pass@4": 0.49605475601264115, + "pass@8": 0.5761246431366777, + "pass@16": 0.6422153424913305, + "pass@32": 0.6975131016174686, + "pass@64": 0.745362531715204, + "pass@128": 0.7885714285714286 + }, + "19": { + "pass@1": 0.21506662912912913, + "pass@2": 0.304116862925918, + "pass@4": 0.3969523421058855, + "pass@8": 0.48389617153847675, + "pass@16": 0.5573617470631065, + "pass@32": 0.6160098154126928, + "pass@64": 0.6654528414928708, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20680837087087087, + "pass@2": 0.29180041261734174, + "pass@4": 0.3802260170744425, + "pass@8": 0.4603380140858725, + "pass@16": 0.5255192306118455, + "pass@32": 0.5771437801720317, + "pass@64": 0.6203514730224098, + "pass@128": 0.6606606606606606 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644354, + "answer_accuracy": 0.7847257404520654, + "avg_response_len": 280.47661364964927, + "resp_tokens_sum": 230304957, + "pass_at_k": { + "pass@1": 0.7847257404520654, + "pass@2": 0.8235093338999934, + "pass@4": 0.8556472382933761, + "pass@8": 0.8813419273206824, + "pass@16": 0.901335019359686, + "pass@32": 0.9169115155133137, + "pass@64": 0.9294425674158958, + "pass@128": 0.9406079501169134 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9608656058282209, + "pass@2": 0.9739629878991349, + "pass@4": 0.9843521879550335, + "pass@8": 0.9903643149049658, + "pass@16": 0.9930190213003309, + "pass@32": 0.9950204756095313, + "pass@64": 0.997522728288262, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.998935688405797, + "pass@2": 0.9998541452698849, + "pass@4": 0.9999957437385544, + "pass@8": 0.9999999968300656, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994627507163324, + "pass@2": 0.9999439484015071, + "pass@4": 0.999999177842741, + "pass@8": 0.9999999999123027, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994806719653179, + "pass@2": 0.9999505740521598, + "pass@4": 0.9999993552106564, + "pass@8": 0.9999999999508568, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995400528169014, + "pass@2": 0.997621506598647, + "pass@4": 0.9990206005059226, + "pass@8": 0.999771949976716, + "pass@16": 0.9999861357166676, + "pass@32": 0.9999999749699509, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962574850299402, + "pass@2": 0.9973448771747843, + "pass@4": 0.9979126474273051, + "pass@8": 0.9985719326395935, + "pass@16": 0.9993543298783014, + "pass@32": 0.9998914962309466, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9690323795180723, + "pass@2": 0.975195812778674, + "pass@4": 0.9779912812103307, + "pass@8": 0.9793962882487052, + "pass@16": 0.9802826047105917, + "pass@32": 0.9814200443385674, + "pass@64": 0.9830661227587516, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9060295846394985, + "pass@2": 0.9250572347147835, + "pass@4": 0.9364256305736081, + "pass@8": 0.9444015269184481, + "pass@16": 0.9506101312135917, + "pass@32": 0.9556181429280108, + "pass@64": 0.9604398982497184, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8360031512605042, + "pass@2": 0.8767200285626062, + "pass@4": 0.9016726128771714, + "pass@8": 0.9169013117277433, + "pass@16": 0.9277848626738392, + "pass@32": 0.937749843120475, + "pass@64": 0.9470916007788368, + "pass@128": 0.957983193277311 + }, + "14": { + "pass@1": 0.8244243421052632, + "pass@2": 0.8814703055210201, + "pass@4": 0.9161977258325157, + "pass@8": 0.9387559253266154, + "pass@16": 0.9528099888394235, + "pass@32": 0.9609495059700786, + "pass@64": 0.9663334643731861, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.718655303030303, + "pass@2": 0.8018458452636601, + "pass@4": 0.8560838676415438, + "pass@8": 0.8931015580401608, + "pass@16": 0.9194444646380041, + "pass@32": 0.9378258206854141, + "pass@64": 0.9487053603010248, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5079015313390314, + "pass@2": 0.6259958190322363, + "pass@4": 0.7192196907544668, + "pass@8": 0.7885741943570277, + "pass@16": 0.8390539563285804, + "pass@32": 0.8758487348681383, + "pass@64": 0.903097786845089, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.44429347826086957, + "pass@2": 0.5645040378784174, + "pass@4": 0.6619254227538325, + "pass@8": 0.7319004490853245, + "pass@16": 0.7814428010063853, + "pass@32": 0.8193750801545913, + "pass@64": 0.8522784760506044, + "pass@128": 0.8819875776397516 + }, + "18": { + "pass@1": 0.3361443014705882, + "pass@2": 0.43848533754052793, + "pass@4": 0.5339143221067955, + "pass@8": 0.6180126229671818, + "pass@16": 0.6884203893465171, + "pass@32": 0.7449961226518379, + "pass@64": 0.7898113276493597, + "pass@128": 0.8294117647058824 + }, + "19": { + "pass@1": 0.22470238095238096, + "pass@2": 0.3165102409073866, + "pass@4": 0.41335825823111383, + "pass@8": 0.5032275614596812, + "pass@16": 0.5792477084209239, + "pass@32": 0.6428269554995669, + "pass@64": 0.6971000675830643, + "pass@128": 0.7470238095238095 + }, + "20": { + "pass@1": 0.17652975299401197, + "pass@2": 0.25829946897543493, + "pass@4": 0.34719286427145707, + "pass@8": 0.4320416991906228, + "pass@16": 0.5046182598137827, + "pass@32": 0.5614345432315808, + "pass@64": 0.6064991139359319, + "pass@128": 0.6497005988023952 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 621465, + "answer_accuracy": 0.7757142215210098, + "avg_response_len": 300.3896863516536, + "resp_tokens_sum": 240657798, + "pass_at_k": { + "pass@1": 0.7757142215210098, + "pass@2": 0.8146817008704238, + "pass@4": 0.8471384863539215, + "pass@8": 0.8730162514421971, + "pass@16": 0.8927849139769056, + "pass@32": 0.907860993801275, + "pass@64": 0.9198730893330967, + "pass@128": 0.9300207700910689 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648554104477612, + "pass@2": 0.9741424521095311, + "pass@4": 0.9812111282731446, + "pass@8": 0.9867068733480728, + "pass@16": 0.9904638808695707, + "pass@32": 0.9925657358955954, + "pass@64": 0.993663351457559, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935202205882353, + "pass@2": 0.9964693579203334, + "pass@4": 0.9984309421800216, + "pass@8": 0.9994897819490819, + "pass@16": 0.9999291658218862, + "pass@32": 0.9999991192943843, + "pass@64": 0.9999999999953015, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987184542586751, + "pass@2": 0.9995241747186965, + "pass@4": 0.9999318992854599, + "pass@8": 0.9999988154015811, + "pass@16": 0.9999999998336015, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997354628164557, + "pass@2": 0.9988105657579986, + "pass@4": 0.9996534721134541, + "pass@8": 0.9999666353241072, + "pass@16": 0.9999997719708628, + "pass@32": 0.9999999999980841, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9888325825825826, + "pass@2": 0.9915145214111751, + "pass@4": 0.9943399777730485, + "pass@8": 0.9970534931226165, + "pass@16": 0.9990479179312053, + "pass@32": 0.9999009978807923, + "pass@64": 0.9999997310208336, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956541398713826, + "pass@2": 0.997711455933362, + "pass@4": 0.9990751527843585, + "pass@8": 0.9998051462035797, + "pass@16": 0.9999910257088084, + "pass@32": 0.9999999919387248, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9627976190476191, + "pass@2": 0.9732404543182105, + "pass@4": 0.9812498622196033, + "pass@8": 0.986813216612229, + "pass@16": 0.9901692265255165, + "pass@32": 0.9921435817941344, + "pass@64": 0.9932621749327728, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9300697629969419, + "pass@2": 0.949016425269089, + "pass@4": 0.9622435455315791, + "pass@8": 0.9726359204775846, + "pass@16": 0.9809736075732047, + "pass@32": 0.9877093113830763, + "pass@64": 0.9915101226717405, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.800733024691358, + "pass@2": 0.851943745747059, + "pass@4": 0.8837995968096578, + "pass@8": 0.9045265423758843, + "pass@16": 0.9194715571232489, + "pass@32": 0.931730883049093, + "pass@64": 0.94148297277955, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7833364243323442, + "pass@2": 0.8476121667562327, + "pass@4": 0.8857191189380257, + "pass@8": 0.9066765449861103, + "pass@16": 0.9178035221402799, + "pass@32": 0.9258789135725163, + "pass@64": 0.9356006496062804, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6760706018518519, + "pass@2": 0.7585867630261495, + "pass@4": 0.8140038160739167, + "pass@8": 0.8504006010198468, + "pass@16": 0.8757978545346746, + "pass@32": 0.8961371542082629, + "pass@64": 0.9140837796750277, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.5234605457227138, + "pass@2": 0.6393311732051187, + "pass@4": 0.7276214389130556, + "pass@8": 0.7916461822267785, + "pass@16": 0.8357722062419856, + "pass@32": 0.866070360809102, + "pass@64": 0.8857701431976893, + "pass@128": 0.8997050147492626 + }, + "17": { + "pass@1": 0.3897072665662651, + "pass@2": 0.4978039619106348, + "pass@4": 0.5914654885006841, + "pass@8": 0.6677335851808504, + "pass@16": 0.7256923939305502, + "pass@32": 0.7676615174147252, + "pass@64": 0.7993500848218832, + "pass@128": 0.8283132530120482 + }, + "18": { + "pass@1": 0.33177923387096775, + "pass@2": 0.4282702565405131, + "pass@4": 0.5144292856538094, + "pass@8": 0.5859698598964601, + "pass@16": 0.6419601282810513, + "pass@32": 0.6859385553447067, + "pass@64": 0.7240848781690444, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.20822318731117825, + "pass@2": 0.30177767502438324, + "pass@4": 0.40263618218417563, + "pass@8": 0.493831614309686, + "pass@16": 0.5657546281867276, + "pass@32": 0.6211074280384756, + "pass@64": 0.6660439944516493, + "pass@128": 0.7039274924471299 + }, + "20": { + "pass@1": 0.18015671921921922, + "pass@2": 0.2612819512425812, + "pass@4": 0.34916571689800047, + "pass@8": 0.43447167600509806, + "pass@16": 0.5114160353927147, + "pass@32": 0.5747150461384618, + "pass@64": 0.625882612427439, + "pass@128": 0.6666666666666666 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7f9a77aaacc54f5d26993114d1bca480fd6e5914 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf82b8b66eb51682043e9e8e88f850f8907c7687939e8ee6b28acba5d40f68d6 +size 2655957865 diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/summary.csv b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..675ed49ad2b2dea13f69d7f1af63f0d0eb24eff4 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op9-12,0.9942239583333333,0.5867765625,0.7797779605263158,0.0,0.0,0.0,200.42143663194443,358.94868359375,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/summary.json b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..06c9c1a0c5131b9857eeb4850691894b209f4b7b --- /dev/null +++ b/results/composition/block4/SDAR-100M-block4-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op9-12", + "id_acc": 0.9942239583333333, + "ood_acc": 0.5867765625, + "total_acc": 0.7797779605263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.42143663194443, + "ood_avg_resp_len": 358.94868359375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 630601, + "answer_accuracy": 0.7787812697597217, + "avg_response_len": 270.9268482749763, + "resp_tokens_sum": 219377055, + "pass_at_k": { + "pass@1": 0.7787812697597217, + "pass@2": 0.8173944791959636, + "pass@4": 0.8491757596357156, + "pass@8": 0.8744109788366741, + "pass@16": 0.8938459117924357, + "pass@32": 0.9090855522082075, + "pass@64": 0.9215615462798628, + "pass@128": 0.9326588681631363 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9670907079646017, + "pass@2": 0.9783562556616267, + "pass@4": 0.9863912442360632, + "pass@8": 0.9916879946510442, + "pass@16": 0.9952884427643433, + "pass@32": 0.9977497176863996, + "pass@64": 0.9994638731503346, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9978271484375, + "pass@2": 0.9994763471948819, + "pass@4": 0.9999680244071053, + "pass@8": 0.9999998940074054, + "pass@16": 0.9999999999996815, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9959325396825397, + "pass@2": 0.9971773059617548, + "pass@4": 0.9978584415043358, + "pass@8": 0.9985893161342536, + "pass@16": 0.9994090073019657, + "pass@32": 0.9999164177993916, + "pass@64": 0.9999995605640043, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998830464071856, + "pass@2": 0.9999963164222737, + "pass@4": 0.9999999985967324, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9980815458579881, + "pass@2": 0.9994394422960443, + "pass@4": 0.9999394495066816, + "pass@8": 0.9999992102443163, + "pass@16": 0.9999999999285286, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999198717948718, + "pass@2": 0.9999499198717948, + "pass@4": 0.9999994531933509, + "pass@8": 0.9999999999711477, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9964788732394366, + "pass@2": 0.9971543334812022, + "pass@4": 0.9971829639252839, + "pass@8": 0.997183098590574, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9505577195467422, + "pass@2": 0.9641180210122462, + "pass@4": 0.9734464953566355, + "pass@8": 0.9799640965041405, + "pass@16": 0.9847543053720864, + "pass@32": 0.9886326398519015, + "pass@64": 0.9918760253150202, + "pass@128": 0.9943342776203966 + }, + "12": { + "pass@1": 0.9008430437853108, + "pass@2": 0.9205685723341787, + "pass@4": 0.9321145698948646, + "pass@8": 0.9403918855715436, + "pass@16": 0.9476097691840198, + "pass@32": 0.955394921972066, + "pass@64": 0.963951534680491, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8344435736677116, + "pass@2": 0.8757532304445487, + "pass@4": 0.9049746909575169, + "pass@8": 0.92617544555909, + "pass@16": 0.9417792383220905, + "pass@32": 0.9544322640829537, + "pass@64": 0.9637782892800076, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8111127336448598, + "pass@2": 0.8645607200677018, + "pass@4": 0.8928378710254671, + "pass@8": 0.909400920399937, + "pass@16": 0.922035389489197, + "pass@32": 0.9330553564330485, + "pass@64": 0.9418133713811047, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.7193370664739884, + "pass@2": 0.7990888557644162, + "pass@4": 0.8498221508438609, + "pass@8": 0.883376836515064, + "pass@16": 0.9057129592960427, + "pass@32": 0.9217041139833264, + "pass@64": 0.9337069258447572, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5192288306451613, + "pass@2": 0.6311301435102868, + "pass@4": 0.7141203666073994, + "pass@8": 0.770769853314818, + "pass@16": 0.8121276516366241, + "pass@32": 0.8450246042301767, + "pass@64": 0.8711062163496095, + "pass@128": 0.8935483870967742 + }, + "17": { + "pass@1": 0.4158688583815029, + "pass@2": 0.5342351139001411, + "pass@4": 0.6372437631531604, + "pass@8": 0.7165662520336787, + "pass@16": 0.7695623182836836, + "pass@32": 0.8039757451384392, + "pass@64": 0.8285737983798428, + "pass@128": 0.8497109826589595 + }, + "18": { + "pass@1": 0.30189732142857145, + "pass@2": 0.4026367407199098, + "pass@4": 0.49605475601264115, + "pass@8": 0.5761246431366777, + "pass@16": 0.6422153424913305, + "pass@32": 0.6975131016174686, + "pass@64": 0.745362531715204, + "pass@128": 0.7885714285714286 + }, + "19": { + "pass@1": 0.21506662912912913, + "pass@2": 0.304116862925918, + "pass@4": 0.3969523421058855, + "pass@8": 0.48389617153847675, + "pass@16": 0.5573617470631065, + "pass@32": 0.6160098154126928, + "pass@64": 0.6654528414928708, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20680837087087087, + "pass@2": 0.29180041261734174, + "pass@4": 0.3802260170744425, + "pass@8": 0.4603380140858725, + "pass@16": 0.5255192306118455, + "pass@32": 0.5771437801720317, + "pass@64": 0.6203514730224098, + "pass@128": 0.6606606606606606 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644354, + "answer_accuracy": 0.7847257404520654, + "avg_response_len": 280.47661364964927, + "resp_tokens_sum": 230304957, + "pass_at_k": { + "pass@1": 0.7847257404520654, + "pass@2": 0.8235093338999934, + "pass@4": 0.8556472382933761, + "pass@8": 0.8813419273206824, + "pass@16": 0.901335019359686, + "pass@32": 0.9169115155133137, + "pass@64": 0.9294425674158958, + "pass@128": 0.9406079501169134 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9608656058282209, + "pass@2": 0.9739629878991349, + "pass@4": 0.9843521879550335, + "pass@8": 0.9903643149049658, + "pass@16": 0.9930190213003309, + "pass@32": 0.9950204756095313, + "pass@64": 0.997522728288262, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.998935688405797, + "pass@2": 0.9998541452698849, + "pass@4": 0.9999957437385544, + "pass@8": 0.9999999968300656, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994627507163324, + "pass@2": 0.9999439484015071, + "pass@4": 0.999999177842741, + "pass@8": 0.9999999999123027, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994806719653179, + "pass@2": 0.9999505740521598, + "pass@4": 0.9999993552106564, + "pass@8": 0.9999999999508568, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995400528169014, + "pass@2": 0.997621506598647, + "pass@4": 0.9990206005059226, + "pass@8": 0.999771949976716, + "pass@16": 0.9999861357166676, + "pass@32": 0.9999999749699509, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9962574850299402, + "pass@2": 0.9973448771747843, + "pass@4": 0.9979126474273051, + "pass@8": 0.9985719326395935, + "pass@16": 0.9993543298783014, + "pass@32": 0.9998914962309466, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9690323795180723, + "pass@2": 0.975195812778674, + "pass@4": 0.9779912812103307, + "pass@8": 0.9793962882487052, + "pass@16": 0.9802826047105917, + "pass@32": 0.9814200443385674, + "pass@64": 0.9830661227587516, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9060295846394985, + "pass@2": 0.9250572347147835, + "pass@4": 0.9364256305736081, + "pass@8": 0.9444015269184481, + "pass@16": 0.9506101312135917, + "pass@32": 0.9556181429280108, + "pass@64": 0.9604398982497184, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8360031512605042, + "pass@2": 0.8767200285626062, + "pass@4": 0.9016726128771714, + "pass@8": 0.9169013117277433, + "pass@16": 0.9277848626738392, + "pass@32": 0.937749843120475, + "pass@64": 0.9470916007788368, + "pass@128": 0.957983193277311 + }, + "14": { + "pass@1": 0.8244243421052632, + "pass@2": 0.8814703055210201, + "pass@4": 0.9161977258325157, + "pass@8": 0.9387559253266154, + "pass@16": 0.9528099888394235, + "pass@32": 0.9609495059700786, + "pass@64": 0.9663334643731861, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.718655303030303, + "pass@2": 0.8018458452636601, + "pass@4": 0.8560838676415438, + "pass@8": 0.8931015580401608, + "pass@16": 0.9194444646380041, + "pass@32": 0.9378258206854141, + "pass@64": 0.9487053603010248, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5079015313390314, + "pass@2": 0.6259958190322363, + "pass@4": 0.7192196907544668, + "pass@8": 0.7885741943570277, + "pass@16": 0.8390539563285804, + "pass@32": 0.8758487348681383, + "pass@64": 0.903097786845089, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.44429347826086957, + "pass@2": 0.5645040378784174, + "pass@4": 0.6619254227538325, + "pass@8": 0.7319004490853245, + "pass@16": 0.7814428010063853, + "pass@32": 0.8193750801545913, + "pass@64": 0.8522784760506044, + "pass@128": 0.8819875776397516 + }, + "18": { + "pass@1": 0.3361443014705882, + "pass@2": 0.43848533754052793, + "pass@4": 0.5339143221067955, + "pass@8": 0.6180126229671818, + "pass@16": 0.6884203893465171, + "pass@32": 0.7449961226518379, + "pass@64": 0.7898113276493597, + "pass@128": 0.8294117647058824 + }, + "19": { + "pass@1": 0.22470238095238096, + "pass@2": 0.3165102409073866, + "pass@4": 0.41335825823111383, + "pass@8": 0.5032275614596812, + "pass@16": 0.5792477084209239, + "pass@32": 0.6428269554995669, + "pass@64": 0.6971000675830643, + "pass@128": 0.7470238095238095 + }, + "20": { + "pass@1": 0.17652975299401197, + "pass@2": 0.25829946897543493, + "pass@4": 0.34719286427145707, + "pass@8": 0.4320416991906228, + "pass@16": 0.5046182598137827, + "pass@32": 0.5614345432315808, + "pass@64": 0.6064991139359319, + "pass@128": 0.6497005988023952 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 621465, + "answer_accuracy": 0.7757142215210098, + "avg_response_len": 300.3896863516536, + "resp_tokens_sum": 240657798, + "pass_at_k": { + "pass@1": 0.7757142215210098, + "pass@2": 0.8146817008704238, + "pass@4": 0.8471384863539215, + "pass@8": 0.8730162514421971, + "pass@16": 0.8927849139769056, + "pass@32": 0.907860993801275, + "pass@64": 0.9198730893330967, + "pass@128": 0.9300207700910689 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648554104477612, + "pass@2": 0.9741424521095311, + "pass@4": 0.9812111282731446, + "pass@8": 0.9867068733480728, + "pass@16": 0.9904638808695707, + "pass@32": 0.9925657358955954, + "pass@64": 0.993663351457559, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935202205882353, + "pass@2": 0.9964693579203334, + "pass@4": 0.9984309421800216, + "pass@8": 0.9994897819490819, + "pass@16": 0.9999291658218862, + "pass@32": 0.9999991192943843, + "pass@64": 0.9999999999953015, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987184542586751, + "pass@2": 0.9995241747186965, + "pass@4": 0.9999318992854599, + "pass@8": 0.9999988154015811, + "pass@16": 0.9999999998336015, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997354628164557, + "pass@2": 0.9988105657579986, + "pass@4": 0.9996534721134541, + "pass@8": 0.9999666353241072, + "pass@16": 0.9999997719708628, + "pass@32": 0.9999999999980841, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9888325825825826, + "pass@2": 0.9915145214111751, + "pass@4": 0.9943399777730485, + "pass@8": 0.9970534931226165, + "pass@16": 0.9990479179312053, + "pass@32": 0.9999009978807923, + "pass@64": 0.9999997310208336, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956541398713826, + "pass@2": 0.997711455933362, + "pass@4": 0.9990751527843585, + "pass@8": 0.9998051462035797, + "pass@16": 0.9999910257088084, + "pass@32": 0.9999999919387248, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9627976190476191, + "pass@2": 0.9732404543182105, + "pass@4": 0.9812498622196033, + "pass@8": 0.986813216612229, + "pass@16": 0.9901692265255165, + "pass@32": 0.9921435817941344, + "pass@64": 0.9932621749327728, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9300697629969419, + "pass@2": 0.949016425269089, + "pass@4": 0.9622435455315791, + "pass@8": 0.9726359204775846, + "pass@16": 0.9809736075732047, + "pass@32": 0.9877093113830763, + "pass@64": 0.9915101226717405, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.800733024691358, + "pass@2": 0.851943745747059, + "pass@4": 0.8837995968096578, + "pass@8": 0.9045265423758843, + "pass@16": 0.9194715571232489, + "pass@32": 0.931730883049093, + "pass@64": 0.94148297277955, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7833364243323442, + "pass@2": 0.8476121667562327, + "pass@4": 0.8857191189380257, + "pass@8": 0.9066765449861103, + "pass@16": 0.9178035221402799, + "pass@32": 0.9258789135725163, + "pass@64": 0.9356006496062804, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6760706018518519, + "pass@2": 0.7585867630261495, + "pass@4": 0.8140038160739167, + "pass@8": 0.8504006010198468, + "pass@16": 0.8757978545346746, + "pass@32": 0.8961371542082629, + "pass@64": 0.9140837796750277, + "pass@128": 0.9290123456790124 + }, + "16": { + "pass@1": 0.5234605457227138, + "pass@2": 0.6393311732051187, + "pass@4": 0.7276214389130556, + "pass@8": 0.7916461822267785, + "pass@16": 0.8357722062419856, + "pass@32": 0.866070360809102, + "pass@64": 0.8857701431976893, + "pass@128": 0.8997050147492626 + }, + "17": { + "pass@1": 0.3897072665662651, + "pass@2": 0.4978039619106348, + "pass@4": 0.5914654885006841, + "pass@8": 0.6677335851808504, + "pass@16": 0.7256923939305502, + "pass@32": 0.7676615174147252, + "pass@64": 0.7993500848218832, + "pass@128": 0.8283132530120482 + }, + "18": { + "pass@1": 0.33177923387096775, + "pass@2": 0.4282702565405131, + "pass@4": 0.5144292856538094, + "pass@8": 0.5859698598964601, + "pass@16": 0.6419601282810513, + "pass@32": 0.6859385553447067, + "pass@64": 0.7240848781690444, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.20822318731117825, + "pass@2": 0.30177767502438324, + "pass@4": 0.40263618218417563, + "pass@8": 0.493831614309686, + "pass@16": 0.5657546281867276, + "pass@32": 0.6211074280384756, + "pass@64": 0.6660439944516493, + "pass@128": 0.7039274924471299 + }, + "20": { + "pass@1": 0.18015671921921922, + "pass@2": 0.2612819512425812, + "pass@4": 0.34916571689800047, + "pass@8": 0.43447167600509806, + "pass@16": 0.5114160353927147, + "pass@32": 0.5747150461384618, + "pass@64": 0.625882612427439, + "pass@128": 0.6666666666666666 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e251db4e864ce2940f5c89af59fc7e91c28a09b4 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69e01419e016da1a8523943c25caf1fc9dc0c59fad7f05cdf754489973f68368 +size 1791758309 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2fda259dcc7eb2c48a5239de2b5158fbd93be68e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op11-14", + "id": { + "answer_accuracy": 0.9810355902777778, + "avg_response_len": 198.64085416666666, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9810355902777778, + "pass@2": 0.9863048993875747, + "pass@4": 0.9894303829208847, + "pass@8": 0.9915208998011644, + "pass@16": 0.9931019961439678, + "pass@32": 0.9943543979195464, + "pass@64": 0.9952654214393646, + "pass@128": 0.996 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.926015625, + "pass@2": 0.9420354330708667, + "pass@4": 0.9523748000562431, + "pass@8": 0.9591922215843824, + "pass@16": 0.9647237045922258, + "pass@32": 0.9697171177569228, + "pass@64": 0.9736260255662195, + "pass@128": 0.977 + }, + "2": { + "pass@1": 0.9994375, + "pass@2": 0.999710999015748, + "pass@4": 0.9999189631608549, + "pass@8": 0.9999941516135135, + "pass@16": 0.9999999795034255, + "pass@32": 0.999999999999968, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.995390625, + "pass@2": 0.9972076771653544, + "pass@4": 0.9986613853580802, + "pass@8": 0.9995636545203327, + "pass@16": 0.9999250524322382, + "pass@32": 0.9999973978839442, + "pass@64": 0.9999999996028255, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9944296875, + "pass@2": 0.9951653543307086, + "pass@4": 0.9957191748218973, + "pass@8": 0.9961200445966201, + "pass@16": 0.9963730891215672, + "pass@32": 0.9967499993046111, + "pass@64": 0.9975, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9794296875, + "pass@2": 0.9850874753937008, + "pass@4": 0.9889421668541432, + "pass@8": 0.9917564351443646, + "pass@16": 0.9941483628117697, + "pass@32": 0.9960428652764948, + "pass@64": 0.9972070762924107, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9928125, + "pass@2": 0.9942552903543309, + "pass@4": 0.9952505364641918, + "pass@8": 0.996359032453305, + "pass@16": 0.9975970511831275, + "pass@32": 0.9985856771554614, + "pass@64": 0.9989718965365927, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9866171875, + "pass@2": 0.9915905511811023, + "pass@4": 0.994014107142857, + "pass@8": 0.995453224954668, + "pass@16": 0.9965368414351305, + "pass@32": 0.9974666982056417, + "pass@64": 0.9983177250156134, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.98221875, + "pass@2": 0.9889499261811026, + "pass@4": 0.9924880063742032, + "pass@8": 0.9947838876781229, + "pass@16": 0.9961119257829661, + "pass@32": 0.9966726961359524, + "pass@64": 0.9969404277573022, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.97296875, + "pass@2": 0.9827413877952758, + "pass@4": 0.9875043060554931, + "pass@8": 0.9904654456651744, + "pass@16": 0.9925019584332531, + "pass@32": 0.9939571295569224, + "pass@64": 0.9948256421833143, + "pass@128": 0.995 + } + }, + "per_op_accuracy": { + "10": 0.926015625, + "2": 0.9994375, + "3": 0.995390625, + "4": 0.9944296875, + "5": 0.9794296875, + "6": 0.9928125, + "7": 0.9866171875, + "8": 0.98221875, + "9": 0.97296875 + }, + "per_op_avg_response_len": { + "10": 277.96453125, + "2": 114.898890625, + "3": 144.083296875, + "4": 159.08728125, + "5": 172.8181484375, + "6": 197.4666953125, + "7": 222.4177265625, + "8": 241.6951484375, + "9": 257.33596875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.96453125, + "2": 114.898890625, + "3": 144.083296875, + "4": 159.08728125, + "5": 172.8181484375, + "6": 197.4666953125, + "7": 222.4177265625, + "8": 241.6951484375, + "9": 257.33596875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371519, + "answer_accuracy": 0.9802405226274907, + "avg_response_len": 189.35646213272543, + "resp_tokens_sum": 71767614, + "pass_at_k": { + "pass@1": 0.9802405226274907, + "pass@2": 0.9860665272559017, + "pass@4": 0.9894760113255863, + "pass@8": 0.9916480224848944, + "pass@16": 0.9931198818928989, + "pass@32": 0.9942258318644791, + "pass@64": 0.9951575585393393, + "pass@128": 0.9959473150962512 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.92765947640118, + "pass@2": 0.9427453516595825, + "pass@4": 0.9518446643284632, + "pass@8": 0.9569369815350387, + "pass@16": 0.9607253064011604, + "pass@32": 0.9644388123872352, + "pass@64": 0.9678451893852689, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940988149847095, + "pass@2": 0.9961130324592453, + "pass@4": 0.9978802145144702, + "pass@8": 0.99912401718979, + "pass@16": 0.9998036701029406, + "pass@32": 0.9999921787847252, + "pass@64": 0.999999998785431, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9863525390625, + "pass@2": 0.9880017378198819, + "pass@4": 0.9896498637865581, + "pass@8": 0.9908048268645558, + "pass@16": 0.9914002785048976, + "pass@32": 0.9921874978269095, + "pass@64": 0.99375, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9736607142857143, + "pass@2": 0.981424587551556, + "pass@4": 0.986885817844198, + "pass@8": 0.9910837111748402, + "pass@16": 0.9947763850873934, + "pass@32": 0.9975508865231105, + "pass@64": 0.9991870600928671, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9953218562874252, + "pass@2": 0.9971481741241923, + "pass@4": 0.9979008389295647, + "pass@8": 0.9985718780083792, + "pass@16": 0.9993543298780693, + "pass@32": 0.9998914962309466, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9878420857988166, + "pass@2": 0.9929559008526301, + "pass@4": 0.9956731027149712, + "pass@8": 0.9972010584778855, + "pass@16": 0.9980041443941395, + "pass@32": 0.9987616712981189, + "pass@64": 0.9996389134790091, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842247596153846, + "pass@2": 0.9897513186452656, + "pass@4": 0.9921717405516617, + "pass@8": 0.9937934100501027, + "pass@16": 0.9948725280340435, + "pass@32": 0.9957963573084669, + "pass@64": 0.9966039773867197, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9752200704225352, + "pass@2": 0.987732546855939, + "pass@4": 0.9940995413425434, + "pass@8": 0.9979300891370281, + "pass@16": 0.9997004651718514, + "pass@32": 0.999995187631867, + "pass@64": 0.999999999928364, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383157, + "answer_accuracy": 0.9808040833879423, + "avg_response_len": 196.57540393594365, + "resp_tokens_sum": 76793361, + "pass_at_k": { + "pass@1": 0.9808040833879423, + "pass@2": 0.9850656210978219, + "pass@4": 0.9877156853693582, + "pass@8": 0.9897168079148662, + "pass@16": 0.9914056337732836, + "pass@32": 0.9927805248243585, + "pass@64": 0.9937986432341875, + "pass@128": 0.9947575360419397 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9177530674846626, + "pass@2": 0.9306849608714556, + "pass@4": 0.9396634427598389, + "pass@8": 0.9465554774872416, + "pass@16": 0.9526982961004865, + "pass@32": 0.9585663594010565, + "pass@64": 0.9637611905177609, + "pass@128": 0.9693251533742331 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9929924242424243, + "pass@2": 0.9957759186351706, + "pass@4": 0.9980904602833736, + "pass@8": 0.9995463133567317, + "pass@16": 0.9999674312702992, + "pass@32": 0.9999998649131489, + "pass@64": 0.9999999999999681, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9967640532544378, + "pass@2": 0.9970709039975773, + "pass@4": 0.9971338174998835, + "pass@8": 0.9972263313608536, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9881340579710145, + "pass@2": 0.9911731142302864, + "pass@4": 0.9931998633323009, + "pass@8": 0.9946199522875911, + "pass@16": 0.9958967610104086, + "pass@32": 0.9971546174071518, + "pass@64": 0.9984671915702185, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9885162965616046, + "pass@2": 0.9899036617557476, + "pass@4": 0.9911227788861635, + "pass@8": 0.9929061104412944, + "pass@16": 0.9949970235706795, + "pass@32": 0.9965411050288688, + "pass@64": 0.9970946298260788, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9884393063583815, + "pass@2": 0.9917522671931183, + "pass@4": 0.9937227489807994, + "pass@8": 0.9954667097667711, + "pass@16": 0.9973801649170918, + "pass@32": 0.9989458768942439, + "pass@64": 0.9998259313864438, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9861355633802817, + "pass@2": 0.9929137323943659, + "pass@4": 0.9964693462612949, + "pass@8": 0.9986503552794669, + "pass@16": 0.9997609671772751, + "pass@32": 0.9999936894028181, + "pass@64": 0.9999999995264219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9660366766467066, + "pass@2": 0.9740694545947474, + "pass@4": 0.977983279784638, + "pass@8": 0.9805549547931032, + "pass@16": 0.9827217761799371, + "pass@32": 0.9843693154907419, + "pass@64": 0.9850009941597133, + "pass@128": 0.9850299401197605 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375477, + "answer_accuracy": 0.9820602820555742, + "avg_response_len": 209.9548276908269, + "resp_tokens_sum": 80273289, + "pass_at_k": { + "pass@1": 0.9820602820555742, + "pass@2": 0.9878074428165096, + "pass@4": 0.9911371627069382, + "pass@8": 0.9932382343075122, + "pass@16": 0.9948175429309576, + "pass@32": 0.9960899669773196, + "pass@64": 0.9968710421722644, + "pass@128": 0.9973217274857716 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.932392723880597, + "pass@2": 0.9523625719826068, + "pass@4": 0.9652811239080186, + "pass@8": 0.973771639352727, + "pass@16": 0.9804721826670864, + "pass@32": 0.9859096334116593, + "pass@64": 0.9890756962860398, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9983928571428572, + "pass@2": 0.9991742829021373, + "pass@4": 0.9997684661738712, + "pass@8": 0.9999832903243243, + "pass@16": 0.9999999414383584, + "pass@32": 0.9999999999999087, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9989294825072886, + "pass@2": 0.999628753357361, + "pass@4": 0.9999554032604525, + "pass@8": 0.9999994679053937, + "pass@16": 0.9999999999690319, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996801900584795, + "pass@2": 0.9999848908689046, + "pass@4": 0.9999999651907547, + "pass@8": 0.9999999999999816, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9759420955882353, + "pass@2": 0.9823058997220936, + "pass@4": 0.9865270040693442, + "pass@8": 0.9894740664266802, + "pass@16": 0.9917924087077937, + "pass@32": 0.9935176265183757, + "pass@64": 0.9940940331512719, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9948984621451105, + "pass@2": 0.9959981774261655, + "pass@4": 0.9970025439446253, + "pass@8": 0.997828998910078, + "pass@16": 0.9986080245352847, + "pass@32": 0.9994607941932178, + "pass@64": 0.9999563921171392, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9833119066455697, + "pass@2": 0.9899530767218181, + "pass@4": 0.9925586306260769, + "pass@8": 0.9935689418032917, + "pass@16": 0.9940439986348026, + "pass@32": 0.9944619617135103, + "pass@64": 0.9952531645569581, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9761636636636637, + "pass@2": 0.9839733926840228, + "pass@4": 0.9885399561541294, + "pass@8": 0.9915899928476881, + "pass@16": 0.9933830441093449, + "pass@32": 0.9939533721252563, + "pass@64": 0.9939939548971943, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9778436495176849, + "pass@2": 0.9863573499253109, + "pass@4": 0.9902011364335085, + "pass@8": 0.9925881322206849, + "pass@16": 0.994788488916871, + "pass@32": 0.9973616931630284, + "pass@64": 0.999470450673314, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.60754453125, + "avg_response_len": 362.74653671875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.60754453125, + "pass@2": 0.6780060900590434, + "pass@4": 0.7330182133577003, + "pass@8": 0.7762659838005682, + "pass@16": 0.8108438728792776, + "pass@32": 0.8390659956603966, + "pass@64": 0.8624079087813213, + "pass@128": 0.8822 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.929734375, + "pass@2": 0.9477073080708667, + "pass@4": 0.9587798506749153, + "pass@8": 0.9662833393698964, + "pass@16": 0.97186013894055, + "pass@32": 0.9769142750599049, + "pass@64": 0.981535836922066, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8863515625, + "pass@2": 0.9109979084645675, + "pass@4": 0.9241851062992124, + "pass@8": 0.933158633230677, + "pass@16": 0.9412422192513017, + "pass@32": 0.9489175311794725, + "pass@64": 0.9547085063946666, + "pass@128": 0.959 + }, + "13": { + "pass@1": 0.796140625, + "pass@2": 0.8443818897637804, + "pass@4": 0.8755496600112485, + "pass@8": 0.8989538471917863, + "pass@16": 0.9177670737845409, + "pass@32": 0.9333620070546804, + "pass@64": 0.9463012929502299, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.75671875, + "pass@2": 0.8186989419291334, + "pass@4": 0.8592196998500184, + "pass@8": 0.8865692663128356, + "pass@16": 0.9059488551898089, + "pass@32": 0.9201768142373378, + "pass@64": 0.9308105971090367, + "pass@128": 0.939 + }, + "15": { + "pass@1": 0.694640625, + "pass@2": 0.7690621309055128, + "pass@4": 0.82238576256093, + "pass@8": 0.8629224012165501, + "pass@16": 0.8934895456649695, + "pass@32": 0.9156558070039178, + "pass@64": 0.9326786266608926, + "pass@128": 0.948 + }, + "16": { + "pass@1": 0.5682421875, + "pass@2": 0.6647921998031497, + "pass@4": 0.7343475513685792, + "pass@8": 0.7839983458721875, + "pass@16": 0.8215243939440204, + "pass@32": 0.8532938180007968, + "pass@64": 0.880477164745469, + "pass@128": 0.903 + }, + "17": { + "pass@1": 0.4653125, + "pass@2": 0.5629958169291341, + "pass@4": 0.6405598996062987, + "pass@8": 0.7004285375779575, + "pass@16": 0.747485774168857, + "pass@32": 0.7844092113406238, + "pass@64": 0.8133049079380261, + "pass@128": 0.836 + }, + "18": { + "pass@1": 0.4120390625, + "pass@2": 0.512470349409449, + "pass@4": 0.5972972604049488, + "pass@8": 0.6648591568014023, + "pass@16": 0.7175278156076398, + "pass@32": 0.7586685568733738, + "pass@64": 0.7909247538704615, + "pass@128": 0.817 + }, + "19": { + "pass@1": 0.2869765625, + "pass@2": 0.37626919291338606, + "pass@4": 0.45976329293213325, + "pass@8": 0.5340057882631002, + "pass@16": 0.597531258153614, + "pass@32": 0.6518599420501288, + "pass@64": 0.6988435480362656, + "pass@128": 0.741 + }, + "20": { + "pass@1": 0.2792890625, + "pass@2": 0.3726851624015747, + "pass@4": 0.45809404986876606, + "pass@8": 0.5314805221693378, + "pass@16": 0.5940616540873473, + "pass@32": 0.6474019938036997, + "pass@64": 0.6944938531860183, + "pass@128": 0.738 + } + }, + "per_op_accuracy": { + "11": 0.929734375, + "12": 0.8863515625, + "13": 0.796140625, + "14": 0.75671875, + "15": 0.694640625, + "16": 0.5682421875, + "17": 0.4653125, + "18": 0.4120390625, + "19": 0.2869765625, + "20": 0.2792890625 + }, + "per_op_avg_response_len": { + "11": 295.7548046875, + "12": 306.648703125, + "13": 325.8592890625, + "14": 349.9366953125, + "15": 368.2075078125, + "16": 372.819296875, + "17": 391.0627578125, + "18": 401.696359375, + "19": 400.8871328125, + "20": 414.5928203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 295.7548046875, + "12": 306.648703125, + "13": 325.8592890625, + "14": 349.9366953125, + "15": 368.2075078125, + "16": 372.819296875, + "17": 391.0627578125, + "18": 401.696359375, + "19": 400.8871328125, + "20": 414.5928203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 263077, + "answer_accuracy": 0.6107842682020802, + "avg_response_len": 343.647102526003, + "resp_tokens_sum": 148015680, + "pass_at_k": { + "pass@1": 0.6107842682020802, + "pass@2": 0.6796983406652582, + "pass@4": 0.7324247003299938, + "pass@8": 0.773641706769493, + "pass@16": 0.8071018836967849, + "pass@32": 0.8351564182847565, + "pass@64": 0.8589946294890234, + "pass@128": 0.8799405646359584 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9292448654390935, + "pass@2": 0.9472170484709242, + "pass@4": 0.9571915625454748, + "pass@8": 0.9635243201969008, + "pass@16": 0.9681905956279837, + "pass@32": 0.9726604975889886, + "pass@64": 0.9771513573471852, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.886762888418079, + "pass@2": 0.9157303761288317, + "pass@4": 0.9311027644319879, + "pass@8": 0.9412101056825519, + "pass@16": 0.9503255963538064, + "pass@32": 0.95966626573379, + "pass@64": 0.9675230101601905, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.8190144984326019, + "pass@2": 0.8606176566781039, + "pass@4": 0.8869597950922281, + "pass@8": 0.9060766167081518, + "pass@16": 0.9221740075596914, + "pass@32": 0.9382278751831253, + "pass@64": 0.9534670546830522, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.7680101246105919, + "pass@2": 0.8273515496602645, + "pass@4": 0.8653282457683441, + "pass@8": 0.890186816823801, + "pass@16": 0.9085657747544327, + "pass@32": 0.9236329949065711, + "pass@64": 0.9352620330737966, + "pass@128": 0.9439252336448598 + }, + "15": { + "pass@1": 0.7025830924855492, + "pass@2": 0.777058750739611, + "pass@4": 0.8293250523959069, + "pass@8": 0.8682926576349029, + "pass@16": 0.897993946117185, + "pass@32": 0.9196130366442065, + "pass@64": 0.9359691528771521, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5715221774193548, + "pass@2": 0.6697977520955036, + "pass@4": 0.7380861158484213, + "pass@8": 0.785292982923536, + "pass@16": 0.8196180281732074, + "pass@32": 0.8489876494448692, + "pass@64": 0.8763803054259582, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.46604046242774566, + "pass@2": 0.5651601116244142, + "pass@4": 0.6435201374755901, + "pass@8": 0.7034030083623097, + "pass@16": 0.7511799489043616, + "pass@32": 0.7868884288848857, + "pass@64": 0.8124260954091077, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.38729910714285715, + "pass@2": 0.4821509420697408, + "pass@4": 0.5600770319781457, + "pass@8": 0.6244385770225844, + "pass@16": 0.6784058987443007, + "pass@32": 0.722883375948966, + "pass@64": 0.7583760698737773, + "pass@128": 0.7885714285714286 + }, + "19": { + "pass@1": 0.28664602102102105, + "pass@2": 0.37364887032701993, + "pass@4": 0.45444786659550424, + "pass@8": 0.5265900892058419, + "pass@16": 0.5875503123933077, + "pass@32": 0.6403612689069341, + "pass@64": 0.6876020080610559, + "pass@128": 0.7297297297297297 + }, + "20": { + "pass@1": 0.2793731231231231, + "pass@2": 0.37029709335319566, + "pass@4": 0.45234337092998506, + "pass@8": 0.5226665018110399, + "pass@16": 0.5828711716738674, + "pass@32": 0.635328191387352, + "pass@64": 0.6837192393407937, + "pass@128": 0.7357357357357357 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 261568, + "answer_accuracy": 0.6076419863217365, + "avg_response_len": 358.927180902468, + "resp_tokens_sum": 154505230, + "pass_at_k": { + "pass@1": 0.6076419863217365, + "pass@2": 0.6800133706664283, + "pass@4": 0.7367404048289683, + "pass@8": 0.7809978299854058, + "pass@16": 0.8159100609299225, + "pass@32": 0.8441975636689191, + "pass@64": 0.8675651867559891, + "pass@128": 0.8876003568242641 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9311229292168675, + "pass@2": 0.9511857716298265, + "pass@4": 0.9648874999435312, + "pass@8": 0.9739992391079406, + "pass@16": 0.979915982826315, + "pass@32": 0.984161196737809, + "pass@64": 0.9874942697215326, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8904045846394985, + "pass@2": 0.9096235313356212, + "pass@4": 0.9183919941041851, + "pass@8": 0.9226891251143094, + "pass@16": 0.9261930707556499, + "pass@32": 0.9306149145301656, + "pass@64": 0.935566242875129, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7789084383753502, + "pass@2": 0.8245119406030129, + "pass@4": 0.8549960517015194, + "pass@8": 0.8804449465320536, + "pass@16": 0.9024942571697264, + "pass@32": 0.9203042288622288, + "pass@64": 0.9338851955900517, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7620842470760234, + "pass@2": 0.8245991763365098, + "pass@4": 0.8647133039291137, + "pass@8": 0.8919086056392456, + "pass@16": 0.9107030652394515, + "pass@32": 0.923205795902636, + "pass@64": 0.9308677176088207, + "pass@128": 0.935672514619883 + }, + "15": { + "pass@1": 0.6909564393939394, + "pass@2": 0.7760987085421143, + "pass@4": 0.8337792863960181, + "pass@8": 0.8751489162287881, + "pass@16": 0.9048666340852678, + "pass@32": 0.9262996212562394, + "pass@64": 0.943582344054327, + "pass@128": 0.9575757575757575 + }, + "16": { + "pass@1": 0.5601851851851852, + "pass@2": 0.6524959620432064, + "pass@4": 0.7218095694768917, + "pass@8": 0.7744315809512314, + "pass@16": 0.81671759750628, + "pass@32": 0.8535284540652478, + "pass@64": 0.8845581322997668, + "pass@128": 0.9088319088319088 + }, + "17": { + "pass@1": 0.480371700310559, + "pass@2": 0.5802058370421087, + "pass@4": 0.6542941344288484, + "pass@8": 0.7080518594968165, + "pass@16": 0.7496651480132744, + "pass@32": 0.7838377390971691, + "pass@64": 0.8123003860358964, + "pass@128": 0.8416149068322981 + }, + "18": { + "pass@1": 0.4148207720588235, + "pass@2": 0.5232587424733671, + "pass@4": 0.6212861383135926, + "pass@8": 0.7001461264509792, + "pass@16": 0.7585386210672321, + "pass@32": 0.8012102356329526, + "pass@64": 0.8329474998597162, + "pass@128": 0.8588235294117647 + }, + "19": { + "pass@1": 0.2847377232142857, + "pass@2": 0.372388510029996, + "pass@4": 0.45478934971298235, + "pass@8": 0.5288187922086712, + "pass@16": 0.5945507632202978, + "pass@32": 0.6528306782846555, + "pass@64": 0.7029838028415101, + "pass@128": 0.7470238095238095 + }, + "20": { + "pass@1": 0.28620883233532934, + "pass@2": 0.3879109399311612, + "pass@4": 0.4794946159424681, + "pass@8": 0.5541569212641627, + "pass@16": 0.6141578134085046, + "pass@32": 0.6638264588754201, + "pass@64": 0.708822484437336, + "pass@128": 0.7485029940119761 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 253012, + "answer_accuracy": 0.6041125458435208, + "avg_response_len": 386.31441253438265, + "resp_tokens_sum": 161794657, + "pass_at_k": { + "pass@1": 0.6041125458435208, + "pass@2": 0.6742026340772613, + "pass@4": 0.7298028837184655, + "pass@8": 0.7741014035713695, + "pass@16": 0.8094851330212991, + "pass@32": 0.8378124090639174, + "pass@64": 0.8606174928246122, + "pass@128": 0.8789731051344744 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9288194444444444, + "pass@2": 0.9445905199350084, + "pass@4": 0.9541224733813035, + "pass@8": 0.9612428792271549, + "pass@16": 0.9674817536048753, + "pass@32": 0.9740431812509169, + "pass@64": 0.9801692388284469, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8819524082568807, + "pass@2": 0.9072154398131429, + "pass@4": 0.9223476500643841, + "pass@8": 0.9346557336623523, + "pass@16": 0.946089812143737, + "pass@32": 0.9551361326439999, + "pass@64": 0.959509875598756, + "pass@128": 0.9602446483180428 + }, + "13": { + "pass@1": 0.7926070601851852, + "pass@2": 0.8502904150870032, + "pass@4": 0.8869626386400771, + "pass@8": 0.9123350634257483, + "pass@16": 0.930256529516691, + "pass@32": 0.9429589665661966, + "pass@64": 0.9529268138292823, + "pass@128": 0.9598765432098766 + }, + "14": { + "pass@1": 0.7405183605341247, + "pass@2": 0.8044693655225587, + "pass@4": 0.8478260623579323, + "pass@8": 0.8777049109310756, + "pass@16": 0.898631433773721, + "pass@32": 0.9138108031858346, + "pass@64": 0.9265125373000923, + "pass@128": 0.9376854599406528 + }, + "15": { + "pass@1": 0.6899112654320988, + "pass@2": 0.7533556460824342, + "pass@4": 0.8033707713619129, + "pass@8": 0.8447345658005373, + "pass@16": 0.8770915156181652, + "pass@32": 0.9005889546619248, + "pass@64": 0.9180590315662641, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5735849926253688, + "pass@2": 0.6729463539126195, + "pass@4": 0.7439106093486095, + "pass@8": 0.7927198709498824, + "pass@16": 0.8282446268012427, + "pass@32": 0.8569886704896351, + "pass@64": 0.8799981286908646, + "pass@128": 0.8997050147492626 + }, + "17": { + "pass@1": 0.44994823042168675, + "pass@2": 0.5440486107817096, + "pass@4": 0.6241542793784808, + "pass@8": 0.6899349335139271, + "pass@16": 0.7415220909267308, + "pass@32": 0.7823797077023043, + "pass@64": 0.8151950440449253, + "pass@128": 0.8373493975903614 + }, + "18": { + "pass@1": 0.43692036290322583, + "pass@2": 0.5348695072390146, + "pass@4": 0.6130097167289579, + "pass@8": 0.6717934575811755, + "pass@16": 0.7167181286589595, + "pass@32": 0.7524125650839734, + "pass@64": 0.7815838047172124, + "pass@128": 0.8032258064516129 + }, + "19": { + "pass@1": 0.28958175981873113, + "pass@2": 0.3828446517353759, + "pass@4": 0.4701599149615357, + "pass@8": 0.546731644638795, + "pass@16": 0.6105980292586793, + "pass@32": 0.6624426936570244, + "pass@64": 0.7059502163056977, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2722644519519519, + "pass@2": 0.359801730864723, + "pass@4": 0.44237989676966044, + "pass@8": 0.5175500461382327, + "pass@16": 0.5850956283528802, + "pass@32": 0.6430020084303948, + "pass@64": 0.6908968068572479, + "pass@128": 0.7297297297297297 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7844613486842106, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7844613486842106, + "pass@2": 0.8240423681620196, + "pass@4": 0.8544766094665599, + "pass@8": 0.8782288387482354, + "pass@16": 0.8971766681099144, + "pass@32": 0.912623659888427, + "pass@64": 0.9253404147772334, + "pass@128": 0.9361052631578948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.926015625, + "pass@2": 0.9420354330708667, + "pass@4": 0.9523748000562431, + "pass@8": 0.9591922215843824, + "pass@16": 0.9647237045922258, + "pass@32": 0.9697171177569228, + "pass@64": 0.9736260255662195, + "pass@128": 0.977 + }, + "2": { + "pass@1": 0.9994375, + "pass@2": 0.999710999015748, + "pass@4": 0.9999189631608549, + "pass@8": 0.9999941516135135, + "pass@16": 0.9999999795034255, + "pass@32": 0.999999999999968, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.995390625, + "pass@2": 0.9972076771653544, + "pass@4": 0.9986613853580802, + "pass@8": 0.9995636545203327, + "pass@16": 0.9999250524322382, + "pass@32": 0.9999973978839442, + "pass@64": 0.9999999996028255, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9944296875, + "pass@2": 0.9951653543307086, + "pass@4": 0.9957191748218973, + "pass@8": 0.9961200445966201, + "pass@16": 0.9963730891215672, + "pass@32": 0.9967499993046111, + "pass@64": 0.9975, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9794296875, + "pass@2": 0.9850874753937008, + "pass@4": 0.9889421668541432, + "pass@8": 0.9917564351443646, + "pass@16": 0.9941483628117697, + "pass@32": 0.9960428652764948, + "pass@64": 0.9972070762924107, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9928125, + "pass@2": 0.9942552903543309, + "pass@4": 0.9952505364641918, + "pass@8": 0.996359032453305, + "pass@16": 0.9975970511831275, + "pass@32": 0.9985856771554614, + "pass@64": 0.9989718965365927, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9866171875, + "pass@2": 0.9915905511811023, + "pass@4": 0.994014107142857, + "pass@8": 0.995453224954668, + "pass@16": 0.9965368414351305, + "pass@32": 0.9974666982056417, + "pass@64": 0.9983177250156134, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.98221875, + "pass@2": 0.9889499261811026, + "pass@4": 0.9924880063742032, + "pass@8": 0.9947838876781229, + "pass@16": 0.9961119257829661, + "pass@32": 0.9966726961359524, + "pass@64": 0.9969404277573022, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.97296875, + "pass@2": 0.9827413877952758, + "pass@4": 0.9875043060554931, + "pass@8": 0.9904654456651744, + "pass@16": 0.9925019584332531, + "pass@32": 0.9939571295569224, + "pass@64": 0.9948256421833143, + "pass@128": 0.995 + }, + "11": { + "pass@1": 0.929734375, + "pass@2": 0.9477073080708667, + "pass@4": 0.9587798506749153, + "pass@8": 0.9662833393698964, + "pass@16": 0.97186013894055, + "pass@32": 0.9769142750599049, + "pass@64": 0.981535836922066, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8863515625, + "pass@2": 0.9109979084645675, + "pass@4": 0.9241851062992124, + "pass@8": 0.933158633230677, + "pass@16": 0.9412422192513017, + "pass@32": 0.9489175311794725, + "pass@64": 0.9547085063946666, + "pass@128": 0.959 + }, + "13": { + "pass@1": 0.796140625, + "pass@2": 0.8443818897637804, + "pass@4": 0.8755496600112485, + "pass@8": 0.8989538471917863, + "pass@16": 0.9177670737845409, + "pass@32": 0.9333620070546804, + "pass@64": 0.9463012929502299, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.75671875, + "pass@2": 0.8186989419291334, + "pass@4": 0.8592196998500184, + "pass@8": 0.8865692663128356, + "pass@16": 0.9059488551898089, + "pass@32": 0.9201768142373378, + "pass@64": 0.9308105971090367, + "pass@128": 0.939 + }, + "15": { + "pass@1": 0.694640625, + "pass@2": 0.7690621309055128, + "pass@4": 0.82238576256093, + "pass@8": 0.8629224012165501, + "pass@16": 0.8934895456649695, + "pass@32": 0.9156558070039178, + "pass@64": 0.9326786266608926, + "pass@128": 0.948 + }, + "16": { + "pass@1": 0.5682421875, + "pass@2": 0.6647921998031497, + "pass@4": 0.7343475513685792, + "pass@8": 0.7839983458721875, + "pass@16": 0.8215243939440204, + "pass@32": 0.8532938180007968, + "pass@64": 0.880477164745469, + "pass@128": 0.903 + }, + "17": { + "pass@1": 0.4653125, + "pass@2": 0.5629958169291341, + "pass@4": 0.6405598996062987, + "pass@8": 0.7004285375779575, + "pass@16": 0.747485774168857, + "pass@32": 0.7844092113406238, + "pass@64": 0.8133049079380261, + "pass@128": 0.836 + }, + "18": { + "pass@1": 0.4120390625, + "pass@2": 0.512470349409449, + "pass@4": 0.5972972604049488, + "pass@8": 0.6648591568014023, + "pass@16": 0.7175278156076398, + "pass@32": 0.7586685568733738, + "pass@64": 0.7909247538704615, + "pass@128": 0.817 + }, + "19": { + "pass@1": 0.2869765625, + "pass@2": 0.37626919291338606, + "pass@4": 0.45976329293213325, + "pass@8": 0.5340057882631002, + "pass@16": 0.597531258153614, + "pass@32": 0.6518599420501288, + "pass@64": 0.6988435480362656, + "pass@128": 0.741 + }, + "20": { + "pass@1": 0.2792890625, + "pass@2": 0.3726851624015747, + "pass@4": 0.45809404986876606, + "pass@8": 0.5314805221693378, + "pass@16": 0.5940616540873473, + "pass@32": 0.6474019938036997, + "pass@64": 0.6944938531860183, + "pass@128": 0.738 + } + }, + "per_op_accuracy": { + "10": 0.926015625, + "2": 0.9994375, + "3": 0.995390625, + "4": 0.9944296875, + "5": 0.9794296875, + "6": 0.9928125, + "7": 0.9866171875, + "8": 0.98221875, + "9": 0.97296875, + "11": 0.929734375, + "12": 0.8863515625, + "13": 0.796140625, + "14": 0.75671875, + "15": 0.694640625, + "16": 0.5682421875, + "17": 0.4653125, + "18": 0.4120390625, + "19": 0.2869765625, + "20": 0.2792890625 + }, + "per_op_avg_response_len": { + "10": 277.96453125, + "2": 114.898890625, + "3": 144.083296875, + "4": 159.08728125, + "5": 172.8181484375, + "6": 197.4666953125, + "7": 222.4177265625, + "8": 241.6951484375, + "9": 257.33596875, + "11": 295.7548046875, + "12": 306.648703125, + "13": 325.8592890625, + "14": 349.9366953125, + "15": 368.2075078125, + "16": 372.819296875, + "17": 391.0627578125, + "18": 401.696359375, + "19": 400.8871328125, + "20": 414.5928203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.96453125, + "2": 114.898890625, + "3": 144.083296875, + "4": 159.08728125, + "5": 172.8181484375, + "6": 197.4666953125, + "7": 222.4177265625, + "8": 241.6951484375, + "9": 257.33596875, + "11": 295.7548046875, + "12": 306.648703125, + "13": 325.8592890625, + "14": 349.9366953125, + "15": 368.2075078125, + "16": 372.819296875, + "17": 391.0627578125, + "18": 401.696359375, + "19": 400.8871328125, + "20": 414.5928203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 634596, + "answer_accuracy": 0.7837150252924439, + "avg_response_len": 271.428546376067, + "resp_tokens_sum": 219783294, + "pass_at_k": { + "pass@1": 0.7837150252924439, + "pass@2": 0.8230995737501184, + "pass@4": 0.8527422678067453, + "pass@8": 0.8756835500880619, + "pass@16": 0.8941710099469781, + "pass@32": 0.9096117666264524, + "pass@64": 0.9227281787963313, + "pass@128": 0.9342396459057857 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.92765947640118, + "pass@2": 0.9427453516595825, + "pass@4": 0.9518446643284632, + "pass@8": 0.9569369815350387, + "pass@16": 0.9607253064011604, + "pass@32": 0.9644388123872352, + "pass@64": 0.9678451893852689, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940988149847095, + "pass@2": 0.9961130324592453, + "pass@4": 0.9978802145144702, + "pass@8": 0.99912401718979, + "pass@16": 0.9998036701029406, + "pass@32": 0.9999921787847252, + "pass@64": 0.999999998785431, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9863525390625, + "pass@2": 0.9880017378198819, + "pass@4": 0.9896498637865581, + "pass@8": 0.9908048268645558, + "pass@16": 0.9914002785048976, + "pass@32": 0.9921874978269095, + "pass@64": 0.99375, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9736607142857143, + "pass@2": 0.981424587551556, + "pass@4": 0.986885817844198, + "pass@8": 0.9910837111748402, + "pass@16": 0.9947763850873934, + "pass@32": 0.9975508865231105, + "pass@64": 0.9991870600928671, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9953218562874252, + "pass@2": 0.9971481741241923, + "pass@4": 0.9979008389295647, + "pass@8": 0.9985718780083792, + "pass@16": 0.9993543298780693, + "pass@32": 0.9998914962309466, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9878420857988166, + "pass@2": 0.9929559008526301, + "pass@4": 0.9956731027149712, + "pass@8": 0.9972010584778855, + "pass@16": 0.9980041443941395, + "pass@32": 0.9987616712981189, + "pass@64": 0.9996389134790091, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842247596153846, + "pass@2": 0.9897513186452656, + "pass@4": 0.9921717405516617, + "pass@8": 0.9937934100501027, + "pass@16": 0.9948725280340435, + "pass@32": 0.9957963573084669, + "pass@64": 0.9966039773867197, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9752200704225352, + "pass@2": 0.987732546855939, + "pass@4": 0.9940995413425434, + "pass@8": 0.9979300891370281, + "pass@16": 0.9997004651718514, + "pass@32": 0.999995187631867, + "pass@64": 0.999999999928364, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9292448654390935, + "pass@2": 0.9472170484709242, + "pass@4": 0.9571915625454748, + "pass@8": 0.9635243201969008, + "pass@16": 0.9681905956279837, + "pass@32": 0.9726604975889886, + "pass@64": 0.9771513573471852, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.886762888418079, + "pass@2": 0.9157303761288317, + "pass@4": 0.9311027644319879, + "pass@8": 0.9412101056825519, + "pass@16": 0.9503255963538064, + "pass@32": 0.95966626573379, + "pass@64": 0.9675230101601905, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.8190144984326019, + "pass@2": 0.8606176566781039, + "pass@4": 0.8869597950922281, + "pass@8": 0.9060766167081518, + "pass@16": 0.9221740075596914, + "pass@32": 0.9382278751831253, + "pass@64": 0.9534670546830522, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.7680101246105919, + "pass@2": 0.8273515496602645, + "pass@4": 0.8653282457683441, + "pass@8": 0.890186816823801, + "pass@16": 0.9085657747544327, + "pass@32": 0.9236329949065711, + "pass@64": 0.9352620330737966, + "pass@128": 0.9439252336448598 + }, + "15": { + "pass@1": 0.7025830924855492, + "pass@2": 0.777058750739611, + "pass@4": 0.8293250523959069, + "pass@8": 0.8682926576349029, + "pass@16": 0.897993946117185, + "pass@32": 0.9196130366442065, + "pass@64": 0.9359691528771521, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5715221774193548, + "pass@2": 0.6697977520955036, + "pass@4": 0.7380861158484213, + "pass@8": 0.785292982923536, + "pass@16": 0.8196180281732074, + "pass@32": 0.8489876494448692, + "pass@64": 0.8763803054259582, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.46604046242774566, + "pass@2": 0.5651601116244142, + "pass@4": 0.6435201374755901, + "pass@8": 0.7034030083623097, + "pass@16": 0.7511799489043616, + "pass@32": 0.7868884288848857, + "pass@64": 0.8124260954091077, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.38729910714285715, + "pass@2": 0.4821509420697408, + "pass@4": 0.5600770319781457, + "pass@8": 0.6244385770225844, + "pass@16": 0.6784058987443007, + "pass@32": 0.722883375948966, + "pass@64": 0.7583760698737773, + "pass@128": 0.7885714285714286 + }, + "19": { + "pass@1": 0.28664602102102105, + "pass@2": 0.37364887032701993, + "pass@4": 0.45444786659550424, + "pass@8": 0.5265900892058419, + "pass@16": 0.5875503123933077, + "pass@32": 0.6403612689069341, + "pass@64": 0.6876020080610559, + "pass@128": 0.7297297297297297 + }, + "20": { + "pass@1": 0.2793731231231231, + "pass@2": 0.37029709335319566, + "pass@4": 0.45234337092998506, + "pass@8": 0.5226665018110399, + "pass@16": 0.5828711716738674, + "pass@32": 0.635328191387352, + "pass@64": 0.6837192393407937, + "pass@128": 0.7357357357357357 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644725, + "answer_accuracy": 0.7851775623538582, + "avg_response_len": 281.68670961613407, + "resp_tokens_sum": 231298591, + "pass_at_k": { + "pass@1": 0.7851775623538582, + "pass@2": 0.825145010310475, + "pass@4": 0.8561443886495829, + "pass@8": 0.8802979579106861, + "pass@16": 0.8994038237230578, + "pass@32": 0.9148873840035109, + "pass@64": 0.9276220081389218, + "pass@128": 0.9385814497272019 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9177530674846626, + "pass@2": 0.9306849608714556, + "pass@4": 0.9396634427598389, + "pass@8": 0.9465554774872416, + "pass@16": 0.9526982961004865, + "pass@32": 0.9585663594010565, + "pass@64": 0.9637611905177609, + "pass@128": 0.9693251533742331 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9929924242424243, + "pass@2": 0.9957759186351706, + "pass@4": 0.9980904602833736, + "pass@8": 0.9995463133567317, + "pass@16": 0.9999674312702992, + "pass@32": 0.9999998649131489, + "pass@64": 0.9999999999999681, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9967640532544378, + "pass@2": 0.9970709039975773, + "pass@4": 0.9971338174998835, + "pass@8": 0.9972263313608536, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9881340579710145, + "pass@2": 0.9911731142302864, + "pass@4": 0.9931998633323009, + "pass@8": 0.9946199522875911, + "pass@16": 0.9958967610104086, + "pass@32": 0.9971546174071518, + "pass@64": 0.9984671915702185, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9885162965616046, + "pass@2": 0.9899036617557476, + "pass@4": 0.9911227788861635, + "pass@8": 0.9929061104412944, + "pass@16": 0.9949970235706795, + "pass@32": 0.9965411050288688, + "pass@64": 0.9970946298260788, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9884393063583815, + "pass@2": 0.9917522671931183, + "pass@4": 0.9937227489807994, + "pass@8": 0.9954667097667711, + "pass@16": 0.9973801649170918, + "pass@32": 0.9989458768942439, + "pass@64": 0.9998259313864438, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9861355633802817, + "pass@2": 0.9929137323943659, + "pass@4": 0.9964693462612949, + "pass@8": 0.9986503552794669, + "pass@16": 0.9997609671772751, + "pass@32": 0.9999936894028181, + "pass@64": 0.9999999995264219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9660366766467066, + "pass@2": 0.9740694545947474, + "pass@4": 0.977983279784638, + "pass@8": 0.9805549547931032, + "pass@16": 0.9827217761799371, + "pass@32": 0.9843693154907419, + "pass@64": 0.9850009941597133, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.9311229292168675, + "pass@2": 0.9511857716298265, + "pass@4": 0.9648874999435312, + "pass@8": 0.9739992391079406, + "pass@16": 0.979915982826315, + "pass@32": 0.984161196737809, + "pass@64": 0.9874942697215326, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8904045846394985, + "pass@2": 0.9096235313356212, + "pass@4": 0.9183919941041851, + "pass@8": 0.9226891251143094, + "pass@16": 0.9261930707556499, + "pass@32": 0.9306149145301656, + "pass@64": 0.935566242875129, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7789084383753502, + "pass@2": 0.8245119406030129, + "pass@4": 0.8549960517015194, + "pass@8": 0.8804449465320536, + "pass@16": 0.9024942571697264, + "pass@32": 0.9203042288622288, + "pass@64": 0.9338851955900517, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7620842470760234, + "pass@2": 0.8245991763365098, + "pass@4": 0.8647133039291137, + "pass@8": 0.8919086056392456, + "pass@16": 0.9107030652394515, + "pass@32": 0.923205795902636, + "pass@64": 0.9308677176088207, + "pass@128": 0.935672514619883 + }, + "15": { + "pass@1": 0.6909564393939394, + "pass@2": 0.7760987085421143, + "pass@4": 0.8337792863960181, + "pass@8": 0.8751489162287881, + "pass@16": 0.9048666340852678, + "pass@32": 0.9262996212562394, + "pass@64": 0.943582344054327, + "pass@128": 0.9575757575757575 + }, + "16": { + "pass@1": 0.5601851851851852, + "pass@2": 0.6524959620432064, + "pass@4": 0.7218095694768917, + "pass@8": 0.7744315809512314, + "pass@16": 0.81671759750628, + "pass@32": 0.8535284540652478, + "pass@64": 0.8845581322997668, + "pass@128": 0.9088319088319088 + }, + "17": { + "pass@1": 0.480371700310559, + "pass@2": 0.5802058370421087, + "pass@4": 0.6542941344288484, + "pass@8": 0.7080518594968165, + "pass@16": 0.7496651480132744, + "pass@32": 0.7838377390971691, + "pass@64": 0.8123003860358964, + "pass@128": 0.8416149068322981 + }, + "18": { + "pass@1": 0.4148207720588235, + "pass@2": 0.5232587424733671, + "pass@4": 0.6212861383135926, + "pass@8": 0.7001461264509792, + "pass@16": 0.7585386210672321, + "pass@32": 0.8012102356329526, + "pass@64": 0.8329474998597162, + "pass@128": 0.8588235294117647 + }, + "19": { + "pass@1": 0.2847377232142857, + "pass@2": 0.372388510029996, + "pass@4": 0.45478934971298235, + "pass@8": 0.5288187922086712, + "pass@16": 0.5945507632202978, + "pass@32": 0.6528306782846555, + "pass@64": 0.7029838028415101, + "pass@128": 0.7470238095238095 + }, + "20": { + "pass@1": 0.28620883233532934, + "pass@2": 0.3879109399311612, + "pass@4": 0.4794946159424681, + "pass@8": 0.5541569212641627, + "pass@16": 0.6141578134085046, + "pass@32": 0.6638264588754201, + "pass@64": 0.708822484437336, + "pass@128": 0.7485029940119761 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 628489, + "answer_accuracy": 0.7844815965010385, + "avg_response_len": 302.1498367351015, + "resp_tokens_sum": 242067946, + "pass_at_k": { + "pass@1": 0.7844815965010385, + "pass@2": 0.823865130275387, + "pass@4": 0.8545201694411919, + "pass@8": 0.8786806835536061, + "pass@16": 0.8979318351143141, + "pass@32": 0.9133476487966757, + "pass@64": 0.9256421536173073, + "pass@128": 0.9354529477552325 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.932392723880597, + "pass@2": 0.9523625719826068, + "pass@4": 0.9652811239080186, + "pass@8": 0.973771639352727, + "pass@16": 0.9804721826670864, + "pass@32": 0.9859096334116593, + "pass@64": 0.9890756962860398, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9983928571428572, + "pass@2": 0.9991742829021373, + "pass@4": 0.9997684661738712, + "pass@8": 0.9999832903243243, + "pass@16": 0.9999999414383584, + "pass@32": 0.9999999999999087, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9989294825072886, + "pass@2": 0.999628753357361, + "pass@4": 0.9999554032604525, + "pass@8": 0.9999994679053937, + "pass@16": 0.9999999999690319, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996801900584795, + "pass@2": 0.9999848908689046, + "pass@4": 0.9999999651907547, + "pass@8": 0.9999999999999816, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9759420955882353, + "pass@2": 0.9823058997220936, + "pass@4": 0.9865270040693442, + "pass@8": 0.9894740664266802, + "pass@16": 0.9917924087077937, + "pass@32": 0.9935176265183757, + "pass@64": 0.9940940331512719, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9948984621451105, + "pass@2": 0.9959981774261655, + "pass@4": 0.9970025439446253, + "pass@8": 0.997828998910078, + "pass@16": 0.9986080245352847, + "pass@32": 0.9994607941932178, + "pass@64": 0.9999563921171392, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9833119066455697, + "pass@2": 0.9899530767218181, + "pass@4": 0.9925586306260769, + "pass@8": 0.9935689418032917, + "pass@16": 0.9940439986348026, + "pass@32": 0.9944619617135103, + "pass@64": 0.9952531645569581, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9761636636636637, + "pass@2": 0.9839733926840228, + "pass@4": 0.9885399561541294, + "pass@8": 0.9915899928476881, + "pass@16": 0.9933830441093449, + "pass@32": 0.9939533721252563, + "pass@64": 0.9939939548971943, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9778436495176849, + "pass@2": 0.9863573499253109, + "pass@4": 0.9902011364335085, + "pass@8": 0.9925881322206849, + "pass@16": 0.994788488916871, + "pass@32": 0.9973616931630284, + "pass@64": 0.999470450673314, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9288194444444444, + "pass@2": 0.9445905199350084, + "pass@4": 0.9541224733813035, + "pass@8": 0.9612428792271549, + "pass@16": 0.9674817536048753, + "pass@32": 0.9740431812509169, + "pass@64": 0.9801692388284469, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8819524082568807, + "pass@2": 0.9072154398131429, + "pass@4": 0.9223476500643841, + "pass@8": 0.9346557336623523, + "pass@16": 0.946089812143737, + "pass@32": 0.9551361326439999, + "pass@64": 0.959509875598756, + "pass@128": 0.9602446483180428 + }, + "13": { + "pass@1": 0.7926070601851852, + "pass@2": 0.8502904150870032, + "pass@4": 0.8869626386400771, + "pass@8": 0.9123350634257483, + "pass@16": 0.930256529516691, + "pass@32": 0.9429589665661966, + "pass@64": 0.9529268138292823, + "pass@128": 0.9598765432098766 + }, + "14": { + "pass@1": 0.7405183605341247, + "pass@2": 0.8044693655225587, + "pass@4": 0.8478260623579323, + "pass@8": 0.8777049109310756, + "pass@16": 0.898631433773721, + "pass@32": 0.9138108031858346, + "pass@64": 0.9265125373000923, + "pass@128": 0.9376854599406528 + }, + "15": { + "pass@1": 0.6899112654320988, + "pass@2": 0.7533556460824342, + "pass@4": 0.8033707713619129, + "pass@8": 0.8447345658005373, + "pass@16": 0.8770915156181652, + "pass@32": 0.9005889546619248, + "pass@64": 0.9180590315662641, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5735849926253688, + "pass@2": 0.6729463539126195, + "pass@4": 0.7439106093486095, + "pass@8": 0.7927198709498824, + "pass@16": 0.8282446268012427, + "pass@32": 0.8569886704896351, + "pass@64": 0.8799981286908646, + "pass@128": 0.8997050147492626 + }, + "17": { + "pass@1": 0.44994823042168675, + "pass@2": 0.5440486107817096, + "pass@4": 0.6241542793784808, + "pass@8": 0.6899349335139271, + "pass@16": 0.7415220909267308, + "pass@32": 0.7823797077023043, + "pass@64": 0.8151950440449253, + "pass@128": 0.8373493975903614 + }, + "18": { + "pass@1": 0.43692036290322583, + "pass@2": 0.5348695072390146, + "pass@4": 0.6130097167289579, + "pass@8": 0.6717934575811755, + "pass@16": 0.7167181286589595, + "pass@32": 0.7524125650839734, + "pass@64": 0.7815838047172124, + "pass@128": 0.8032258064516129 + }, + "19": { + "pass@1": 0.28958175981873113, + "pass@2": 0.3828446517353759, + "pass@4": 0.4701599149615357, + "pass@8": 0.546731644638795, + "pass@16": 0.6105980292586793, + "pass@32": 0.6624426936570244, + "pass@64": 0.7059502163056977, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2722644519519519, + "pass@2": 0.359801730864723, + "pass@4": 0.44237989676966044, + "pass@8": 0.5175500461382327, + "pass@16": 0.5850956283528802, + "pass@32": 0.6430020084303948, + "pass@64": 0.6908968068572479, + "pass@128": 0.7297297297297297 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ec7573bee26e35fe1c4bb9636d2add2576a8e7e3 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ec583a6cbd207b655edba2f38012992d458647d5411e69effc015acc3d0a4e2 +size 2683698878 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/summary.csv b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..6cd330f32a82f7e2e649776b28325a7fd786adda --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op11-14,0.9810355902777778,0.60754453125,0.7844613486842106,0.0,0.0,0.0,198.64085416666666,362.74653671875,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/summary.json b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..846a85d30d781f71f92dbca6eb782e7d6fdd752c --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op11-14", + "id_acc": 0.9810355902777778, + "ood_acc": 0.60754453125, + "total_acc": 0.7844613486842106, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.64085416666666, + "ood_avg_resp_len": 362.74653671875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 634596, + "answer_accuracy": 0.7837150252924439, + "avg_response_len": 271.428546376067, + "resp_tokens_sum": 219783294, + "pass_at_k": { + "pass@1": 0.7837150252924439, + "pass@2": 0.8230995737501184, + "pass@4": 0.8527422678067453, + "pass@8": 0.8756835500880619, + "pass@16": 0.8941710099469781, + "pass@32": 0.9096117666264524, + "pass@64": 0.9227281787963313, + "pass@128": 0.9342396459057857 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.92765947640118, + "pass@2": 0.9427453516595825, + "pass@4": 0.9518446643284632, + "pass@8": 0.9569369815350387, + "pass@16": 0.9607253064011604, + "pass@32": 0.9644388123872352, + "pass@64": 0.9678451893852689, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940988149847095, + "pass@2": 0.9961130324592453, + "pass@4": 0.9978802145144702, + "pass@8": 0.99912401718979, + "pass@16": 0.9998036701029406, + "pass@32": 0.9999921787847252, + "pass@64": 0.999999998785431, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9863525390625, + "pass@2": 0.9880017378198819, + "pass@4": 0.9896498637865581, + "pass@8": 0.9908048268645558, + "pass@16": 0.9914002785048976, + "pass@32": 0.9921874978269095, + "pass@64": 0.99375, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9736607142857143, + "pass@2": 0.981424587551556, + "pass@4": 0.986885817844198, + "pass@8": 0.9910837111748402, + "pass@16": 0.9947763850873934, + "pass@32": 0.9975508865231105, + "pass@64": 0.9991870600928671, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9953218562874252, + "pass@2": 0.9971481741241923, + "pass@4": 0.9979008389295647, + "pass@8": 0.9985718780083792, + "pass@16": 0.9993543298780693, + "pass@32": 0.9998914962309466, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9878420857988166, + "pass@2": 0.9929559008526301, + "pass@4": 0.9956731027149712, + "pass@8": 0.9972010584778855, + "pass@16": 0.9980041443941395, + "pass@32": 0.9987616712981189, + "pass@64": 0.9996389134790091, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842247596153846, + "pass@2": 0.9897513186452656, + "pass@4": 0.9921717405516617, + "pass@8": 0.9937934100501027, + "pass@16": 0.9948725280340435, + "pass@32": 0.9957963573084669, + "pass@64": 0.9966039773867197, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9752200704225352, + "pass@2": 0.987732546855939, + "pass@4": 0.9940995413425434, + "pass@8": 0.9979300891370281, + "pass@16": 0.9997004651718514, + "pass@32": 0.999995187631867, + "pass@64": 0.999999999928364, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9292448654390935, + "pass@2": 0.9472170484709242, + "pass@4": 0.9571915625454748, + "pass@8": 0.9635243201969008, + "pass@16": 0.9681905956279837, + "pass@32": 0.9726604975889886, + "pass@64": 0.9771513573471852, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.886762888418079, + "pass@2": 0.9157303761288317, + "pass@4": 0.9311027644319879, + "pass@8": 0.9412101056825519, + "pass@16": 0.9503255963538064, + "pass@32": 0.95966626573379, + "pass@64": 0.9675230101601905, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.8190144984326019, + "pass@2": 0.8606176566781039, + "pass@4": 0.8869597950922281, + "pass@8": 0.9060766167081518, + "pass@16": 0.9221740075596914, + "pass@32": 0.9382278751831253, + "pass@64": 0.9534670546830522, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.7680101246105919, + "pass@2": 0.8273515496602645, + "pass@4": 0.8653282457683441, + "pass@8": 0.890186816823801, + "pass@16": 0.9085657747544327, + "pass@32": 0.9236329949065711, + "pass@64": 0.9352620330737966, + "pass@128": 0.9439252336448598 + }, + "15": { + "pass@1": 0.7025830924855492, + "pass@2": 0.777058750739611, + "pass@4": 0.8293250523959069, + "pass@8": 0.8682926576349029, + "pass@16": 0.897993946117185, + "pass@32": 0.9196130366442065, + "pass@64": 0.9359691528771521, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5715221774193548, + "pass@2": 0.6697977520955036, + "pass@4": 0.7380861158484213, + "pass@8": 0.785292982923536, + "pass@16": 0.8196180281732074, + "pass@32": 0.8489876494448692, + "pass@64": 0.8763803054259582, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.46604046242774566, + "pass@2": 0.5651601116244142, + "pass@4": 0.6435201374755901, + "pass@8": 0.7034030083623097, + "pass@16": 0.7511799489043616, + "pass@32": 0.7868884288848857, + "pass@64": 0.8124260954091077, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.38729910714285715, + "pass@2": 0.4821509420697408, + "pass@4": 0.5600770319781457, + "pass@8": 0.6244385770225844, + "pass@16": 0.6784058987443007, + "pass@32": 0.722883375948966, + "pass@64": 0.7583760698737773, + "pass@128": 0.7885714285714286 + }, + "19": { + "pass@1": 0.28664602102102105, + "pass@2": 0.37364887032701993, + "pass@4": 0.45444786659550424, + "pass@8": 0.5265900892058419, + "pass@16": 0.5875503123933077, + "pass@32": 0.6403612689069341, + "pass@64": 0.6876020080610559, + "pass@128": 0.7297297297297297 + }, + "20": { + "pass@1": 0.2793731231231231, + "pass@2": 0.37029709335319566, + "pass@4": 0.45234337092998506, + "pass@8": 0.5226665018110399, + "pass@16": 0.5828711716738674, + "pass@32": 0.635328191387352, + "pass@64": 0.6837192393407937, + "pass@128": 0.7357357357357357 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644725, + "answer_accuracy": 0.7851775623538582, + "avg_response_len": 281.68670961613407, + "resp_tokens_sum": 231298591, + "pass_at_k": { + "pass@1": 0.7851775623538582, + "pass@2": 0.825145010310475, + "pass@4": 0.8561443886495829, + "pass@8": 0.8802979579106861, + "pass@16": 0.8994038237230578, + "pass@32": 0.9148873840035109, + "pass@64": 0.9276220081389218, + "pass@128": 0.9385814497272019 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9177530674846626, + "pass@2": 0.9306849608714556, + "pass@4": 0.9396634427598389, + "pass@8": 0.9465554774872416, + "pass@16": 0.9526982961004865, + "pass@32": 0.9585663594010565, + "pass@64": 0.9637611905177609, + "pass@128": 0.9693251533742331 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9929924242424243, + "pass@2": 0.9957759186351706, + "pass@4": 0.9980904602833736, + "pass@8": 0.9995463133567317, + "pass@16": 0.9999674312702992, + "pass@32": 0.9999998649131489, + "pass@64": 0.9999999999999681, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9967640532544378, + "pass@2": 0.9970709039975773, + "pass@4": 0.9971338174998835, + "pass@8": 0.9972263313608536, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9881340579710145, + "pass@2": 0.9911731142302864, + "pass@4": 0.9931998633323009, + "pass@8": 0.9946199522875911, + "pass@16": 0.9958967610104086, + "pass@32": 0.9971546174071518, + "pass@64": 0.9984671915702185, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9885162965616046, + "pass@2": 0.9899036617557476, + "pass@4": 0.9911227788861635, + "pass@8": 0.9929061104412944, + "pass@16": 0.9949970235706795, + "pass@32": 0.9965411050288688, + "pass@64": 0.9970946298260788, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9884393063583815, + "pass@2": 0.9917522671931183, + "pass@4": 0.9937227489807994, + "pass@8": 0.9954667097667711, + "pass@16": 0.9973801649170918, + "pass@32": 0.9989458768942439, + "pass@64": 0.9998259313864438, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9861355633802817, + "pass@2": 0.9929137323943659, + "pass@4": 0.9964693462612949, + "pass@8": 0.9986503552794669, + "pass@16": 0.9997609671772751, + "pass@32": 0.9999936894028181, + "pass@64": 0.9999999995264219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9660366766467066, + "pass@2": 0.9740694545947474, + "pass@4": 0.977983279784638, + "pass@8": 0.9805549547931032, + "pass@16": 0.9827217761799371, + "pass@32": 0.9843693154907419, + "pass@64": 0.9850009941597133, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.9311229292168675, + "pass@2": 0.9511857716298265, + "pass@4": 0.9648874999435312, + "pass@8": 0.9739992391079406, + "pass@16": 0.979915982826315, + "pass@32": 0.984161196737809, + "pass@64": 0.9874942697215326, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8904045846394985, + "pass@2": 0.9096235313356212, + "pass@4": 0.9183919941041851, + "pass@8": 0.9226891251143094, + "pass@16": 0.9261930707556499, + "pass@32": 0.9306149145301656, + "pass@64": 0.935566242875129, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7789084383753502, + "pass@2": 0.8245119406030129, + "pass@4": 0.8549960517015194, + "pass@8": 0.8804449465320536, + "pass@16": 0.9024942571697264, + "pass@32": 0.9203042288622288, + "pass@64": 0.9338851955900517, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7620842470760234, + "pass@2": 0.8245991763365098, + "pass@4": 0.8647133039291137, + "pass@8": 0.8919086056392456, + "pass@16": 0.9107030652394515, + "pass@32": 0.923205795902636, + "pass@64": 0.9308677176088207, + "pass@128": 0.935672514619883 + }, + "15": { + "pass@1": 0.6909564393939394, + "pass@2": 0.7760987085421143, + "pass@4": 0.8337792863960181, + "pass@8": 0.8751489162287881, + "pass@16": 0.9048666340852678, + "pass@32": 0.9262996212562394, + "pass@64": 0.943582344054327, + "pass@128": 0.9575757575757575 + }, + "16": { + "pass@1": 0.5601851851851852, + "pass@2": 0.6524959620432064, + "pass@4": 0.7218095694768917, + "pass@8": 0.7744315809512314, + "pass@16": 0.81671759750628, + "pass@32": 0.8535284540652478, + "pass@64": 0.8845581322997668, + "pass@128": 0.9088319088319088 + }, + "17": { + "pass@1": 0.480371700310559, + "pass@2": 0.5802058370421087, + "pass@4": 0.6542941344288484, + "pass@8": 0.7080518594968165, + "pass@16": 0.7496651480132744, + "pass@32": 0.7838377390971691, + "pass@64": 0.8123003860358964, + "pass@128": 0.8416149068322981 + }, + "18": { + "pass@1": 0.4148207720588235, + "pass@2": 0.5232587424733671, + "pass@4": 0.6212861383135926, + "pass@8": 0.7001461264509792, + "pass@16": 0.7585386210672321, + "pass@32": 0.8012102356329526, + "pass@64": 0.8329474998597162, + "pass@128": 0.8588235294117647 + }, + "19": { + "pass@1": 0.2847377232142857, + "pass@2": 0.372388510029996, + "pass@4": 0.45478934971298235, + "pass@8": 0.5288187922086712, + "pass@16": 0.5945507632202978, + "pass@32": 0.6528306782846555, + "pass@64": 0.7029838028415101, + "pass@128": 0.7470238095238095 + }, + "20": { + "pass@1": 0.28620883233532934, + "pass@2": 0.3879109399311612, + "pass@4": 0.4794946159424681, + "pass@8": 0.5541569212641627, + "pass@16": 0.6141578134085046, + "pass@32": 0.6638264588754201, + "pass@64": 0.708822484437336, + "pass@128": 0.7485029940119761 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 628489, + "answer_accuracy": 0.7844815965010385, + "avg_response_len": 302.1498367351015, + "resp_tokens_sum": 242067946, + "pass_at_k": { + "pass@1": 0.7844815965010385, + "pass@2": 0.823865130275387, + "pass@4": 0.8545201694411919, + "pass@8": 0.8786806835536061, + "pass@16": 0.8979318351143141, + "pass@32": 0.9133476487966757, + "pass@64": 0.9256421536173073, + "pass@128": 0.9354529477552325 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.932392723880597, + "pass@2": 0.9523625719826068, + "pass@4": 0.9652811239080186, + "pass@8": 0.973771639352727, + "pass@16": 0.9804721826670864, + "pass@32": 0.9859096334116593, + "pass@64": 0.9890756962860398, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9983928571428572, + "pass@2": 0.9991742829021373, + "pass@4": 0.9997684661738712, + "pass@8": 0.9999832903243243, + "pass@16": 0.9999999414383584, + "pass@32": 0.9999999999999087, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9989294825072886, + "pass@2": 0.999628753357361, + "pass@4": 0.9999554032604525, + "pass@8": 0.9999994679053937, + "pass@16": 0.9999999999690319, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996801900584795, + "pass@2": 0.9999848908689046, + "pass@4": 0.9999999651907547, + "pass@8": 0.9999999999999816, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9759420955882353, + "pass@2": 0.9823058997220936, + "pass@4": 0.9865270040693442, + "pass@8": 0.9894740664266802, + "pass@16": 0.9917924087077937, + "pass@32": 0.9935176265183757, + "pass@64": 0.9940940331512719, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9948984621451105, + "pass@2": 0.9959981774261655, + "pass@4": 0.9970025439446253, + "pass@8": 0.997828998910078, + "pass@16": 0.9986080245352847, + "pass@32": 0.9994607941932178, + "pass@64": 0.9999563921171392, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9833119066455697, + "pass@2": 0.9899530767218181, + "pass@4": 0.9925586306260769, + "pass@8": 0.9935689418032917, + "pass@16": 0.9940439986348026, + "pass@32": 0.9944619617135103, + "pass@64": 0.9952531645569581, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9761636636636637, + "pass@2": 0.9839733926840228, + "pass@4": 0.9885399561541294, + "pass@8": 0.9915899928476881, + "pass@16": 0.9933830441093449, + "pass@32": 0.9939533721252563, + "pass@64": 0.9939939548971943, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9778436495176849, + "pass@2": 0.9863573499253109, + "pass@4": 0.9902011364335085, + "pass@8": 0.9925881322206849, + "pass@16": 0.994788488916871, + "pass@32": 0.9973616931630284, + "pass@64": 0.999470450673314, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9288194444444444, + "pass@2": 0.9445905199350084, + "pass@4": 0.9541224733813035, + "pass@8": 0.9612428792271549, + "pass@16": 0.9674817536048753, + "pass@32": 0.9740431812509169, + "pass@64": 0.9801692388284469, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8819524082568807, + "pass@2": 0.9072154398131429, + "pass@4": 0.9223476500643841, + "pass@8": 0.9346557336623523, + "pass@16": 0.946089812143737, + "pass@32": 0.9551361326439999, + "pass@64": 0.959509875598756, + "pass@128": 0.9602446483180428 + }, + "13": { + "pass@1": 0.7926070601851852, + "pass@2": 0.8502904150870032, + "pass@4": 0.8869626386400771, + "pass@8": 0.9123350634257483, + "pass@16": 0.930256529516691, + "pass@32": 0.9429589665661966, + "pass@64": 0.9529268138292823, + "pass@128": 0.9598765432098766 + }, + "14": { + "pass@1": 0.7405183605341247, + "pass@2": 0.8044693655225587, + "pass@4": 0.8478260623579323, + "pass@8": 0.8777049109310756, + "pass@16": 0.898631433773721, + "pass@32": 0.9138108031858346, + "pass@64": 0.9265125373000923, + "pass@128": 0.9376854599406528 + }, + "15": { + "pass@1": 0.6899112654320988, + "pass@2": 0.7533556460824342, + "pass@4": 0.8033707713619129, + "pass@8": 0.8447345658005373, + "pass@16": 0.8770915156181652, + "pass@32": 0.9005889546619248, + "pass@64": 0.9180590315662641, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5735849926253688, + "pass@2": 0.6729463539126195, + "pass@4": 0.7439106093486095, + "pass@8": 0.7927198709498824, + "pass@16": 0.8282446268012427, + "pass@32": 0.8569886704896351, + "pass@64": 0.8799981286908646, + "pass@128": 0.8997050147492626 + }, + "17": { + "pass@1": 0.44994823042168675, + "pass@2": 0.5440486107817096, + "pass@4": 0.6241542793784808, + "pass@8": 0.6899349335139271, + "pass@16": 0.7415220909267308, + "pass@32": 0.7823797077023043, + "pass@64": 0.8151950440449253, + "pass@128": 0.8373493975903614 + }, + "18": { + "pass@1": 0.43692036290322583, + "pass@2": 0.5348695072390146, + "pass@4": 0.6130097167289579, + "pass@8": 0.6717934575811755, + "pass@16": 0.7167181286589595, + "pass@32": 0.7524125650839734, + "pass@64": 0.7815838047172124, + "pass@128": 0.8032258064516129 + }, + "19": { + "pass@1": 0.28958175981873113, + "pass@2": 0.3828446517353759, + "pass@4": 0.4701599149615357, + "pass@8": 0.546731644638795, + "pass@16": 0.6105980292586793, + "pass@32": 0.6624426936570244, + "pass@64": 0.7059502163056977, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2722644519519519, + "pass@2": 0.359801730864723, + "pass@4": 0.44237989676966044, + "pass@8": 0.5175500461382327, + "pass@16": 0.5850956283528802, + "pass@32": 0.6430020084303948, + "pass@64": 0.6908968068572479, + "pass@128": 0.7297297297297297 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..34292965e1acf46f9ec34cf380f44e24ca6f3336 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe35db7d412af78a16ac6066b833f4b4d83130f13f994dd4d0e73f1299f28348 +size 1792814243 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3f1afe8bd92fb924c15b3b4825f530ff024a9d9b --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9791267361111111, + "avg_response_len": 199.5855390625, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9791267361111111, + "pass@2": 0.9868001968503884, + "pass@4": 0.9911531453464152, + "pass@8": 0.9940186117368351, + "pass@16": 0.9959945525373363, + "pass@32": 0.997373125527829, + "pass@64": 0.9983448955086357, + "pass@128": 0.9988888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.923390625, + "pass@2": 0.9501561269685036, + "pass@4": 0.9678725301837272, + "pass@8": 0.9793842899999533, + "pass@16": 0.9867344327859435, + "pass@32": 0.9911755389493582, + "pass@64": 0.9936929318542894, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9941796875, + "pass@2": 0.9963764763779527, + "pass@4": 0.9981032145669293, + "pass@8": 0.9992212404213301, + "pass@16": 0.9997815831650843, + "pass@32": 0.9999734313592589, + "pass@64": 0.9999998615770528, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9948125, + "pass@2": 0.9956371801181103, + "pass@4": 0.9962996703224597, + "pass@8": 0.9969636643709774, + "pass@16": 0.9975697588774716, + "pass@32": 0.998123944796517, + "pass@64": 0.9986924084990753, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.975609375, + "pass@2": 0.982559916338583, + "pass@4": 0.9875855872703415, + "pass@8": 0.9912197676299933, + "pass@16": 0.9941905164201755, + "pass@32": 0.9969005728810001, + "pass@64": 0.999023905599881, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992828125, + "pass@2": 0.9953348917322836, + "pass@4": 0.9967678327709036, + "pass@8": 0.9977780583091089, + "pass@16": 0.9985466233300656, + "pass@32": 0.9989439643112072, + "pass@64": 0.9989996702871723, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9856484375, + "pass@2": 0.9913272637795276, + "pass@4": 0.9937962862767151, + "pass@8": 0.9956409850917508, + "pass@16": 0.9971005287140816, + "pass@32": 0.9982284708545968, + "pass@64": 0.9992420693192129, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9763125, + "pass@2": 0.9866906988188983, + "pass@4": 0.9916709403824518, + "pass@8": 0.9949062794665869, + "pass@16": 0.9968789403578321, + "pass@32": 0.9977548704817156, + "pass@64": 0.997985488925269, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.969359375, + "pass@2": 0.9831192175196861, + "pass@4": 0.988282246344207, + "pass@8": 0.9910532203418113, + "pass@16": 0.9931485891853677, + "pass@32": 0.9952573361168118, + "pass@64": 0.9974677235157687, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.923390625, + "2": 1.0, + "3": 0.9941796875, + "4": 0.9948125, + "5": 0.975609375, + "6": 0.992828125, + "7": 0.9856484375, + "8": 0.9763125, + "9": 0.969359375 + }, + "per_op_avg_response_len": { + "10": 279.618734375, + "2": 115.0162734375, + "3": 144.9221328125, + "4": 159.417390625, + "5": 174.09465625, + "6": 198.336203125, + "7": 223.039484375, + "8": 243.111015625, + "9": 258.7139609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.618734375, + "2": 115.0162734375, + "3": 144.9221328125, + "4": 159.417390625, + "5": 174.09465625, + "6": 198.336203125, + "7": 223.039484375, + "8": 243.111015625, + "9": 258.7139609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 370530, + "answer_accuracy": 0.9776310790273556, + "avg_response_len": 190.3361617696724, + "resp_tokens_sum": 72138928, + "pass_at_k": { + "pass@1": 0.9776310790273556, + "pass@2": 0.9860302535321392, + "pass@4": 0.9907222220322772, + "pass@8": 0.9936489055458044, + "pass@16": 0.9955436125918403, + "pass@32": 0.9970057947296482, + "pass@64": 0.997994815006437, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924940081120944, + "pass@2": 0.9509071667479613, + "pass@4": 0.96726450316498, + "pass@8": 0.9770110038469436, + "pass@16": 0.9828349449512709, + "pass@32": 0.9871731776191788, + "pass@64": 0.9900107432528562, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942660550458715, + "pass@2": 0.9962657871607791, + "pass@4": 0.9981546572733454, + "pass@8": 0.9994911036229956, + "pass@16": 0.9999636877139981, + "pass@32": 0.9999998965218795, + "pass@64": 0.9999999999999889, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9873779296875, + "pass@2": 0.9894150621309056, + "pass@4": 0.9913685054016685, + "pass@8": 0.99325659141523, + "pass@16": 0.9947953833031221, + "pass@32": 0.9958905263080137, + "pass@64": 0.9966888749848067, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.966468253968254, + "pass@2": 0.9759350393700784, + "pass@4": 0.9829707724034493, + "pass@8": 0.9879597908973488, + "pass@16": 0.9924307677634848, + "pass@32": 0.9968554804125532, + "pass@64": 0.9996607862988176, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9938014595808383, + "pass@2": 0.9963425756754208, + "pass@4": 0.997830821203487, + "pass@8": 0.9988929507572979, + "pass@16": 0.9996441380031399, + "pass@32": 0.9999703661222745, + "pass@64": 0.9999999638637934, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842825443786982, + "pass@2": 0.9909884889111497, + "pass@4": 0.9934844918608546, + "pass@8": 0.9952204293474521, + "pass@16": 0.9966938224900305, + "pass@32": 0.9982559181300668, + "pass@64": 0.9995980149616418, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9789162660256411, + "pass@2": 0.9889677815213003, + "pass@4": 0.9928349941834194, + "pass@8": 0.9948939475896649, + "pass@16": 0.9959889984825363, + "pass@32": 0.9966354953643705, + "pass@64": 0.9967927316172044, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9708406690140845, + "pass@2": 0.9864121243207277, + "pass@4": 0.9930078146041601, + "pass@8": 0.9962900913419857, + "pass@16": 0.9976705029288833, + "pass@32": 0.998414543391346, + "pass@64": 0.9993013192675217, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382789, + "answer_accuracy": 0.9798620781454783, + "avg_response_len": 197.55019249672347, + "resp_tokens_sum": 77174168, + "pass_at_k": { + "pass@1": 0.9798620781454783, + "pass@2": 0.9868867855465897, + "pass@4": 0.9910125943464152, + "pass@8": 0.9937899797117418, + "pass@16": 0.9957900364699863, + "pass@32": 0.997199592310623, + "pass@64": 0.9982626243798648, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9208445168711656, + "pass@2": 0.9478686114197385, + "pass@4": 0.9663925215828081, + "pass@8": 0.978914794263826, + "pass@16": 0.987045205518265, + "pass@32": 0.9917305217541144, + "pass@64": 0.9945764355971969, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9903409090909091, + "pass@2": 0.9936258500357912, + "pass@4": 0.9963124410585041, + "pass@8": 0.9981600601561137, + "pass@16": 0.9993741606859169, + "pass@32": 0.999919591504913, + "pass@64": 0.9999995805365348, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968333949704142, + "pass@2": 0.9971200437963007, + "pass@4": 0.9972241124260354, + "pass@8": 0.9974010506452778, + "pass@16": 0.9977373852676699, + "pass@32": 0.9983401667986769, + "pass@64": 0.9992661790057308, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9831295289855072, + "pass@2": 0.9876786631290653, + "pass@4": 0.9912679836216123, + "pass@8": 0.994079110382506, + "pass@16": 0.9965939665801291, + "pass@32": 0.9984886256195019, + "pass@64": 0.9996372533496305, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9873298710601719, + "pass@2": 0.9903450240281569, + "pass@4": 0.992818670946934, + "pass@8": 0.9946928744639584, + "pass@16": 0.9961761640029136, + "pass@32": 0.9970024699895921, + "pass@64": 0.9971337603342841, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.990606936416185, + "pass@2": 0.9943277502161942, + "pass@4": 0.9956170541904806, + "pass@8": 0.9965211697808511, + "pass@16": 0.9972863841958547, + "pass@32": 0.997824978258472, + "pass@64": 0.9985549121469247, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9817341549295775, + "pass@2": 0.9913302650548964, + "pass@4": 0.9955078983084861, + "pass@8": 0.9976091236030408, + "pass@16": 0.9987230947253419, + "pass@32": 0.9995183835954631, + "pass@64": 0.9999610600031621, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658027694610778, + "pass@2": 0.9781766437597247, + "pass@4": 0.9829689976065864, + "pass@8": 0.9860890727169717, + "pass@16": 0.9887651430963902, + "pass@32": 0.9916995734930811, + "pass@64": 0.9950690744174322, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 374635, + "answer_accuracy": 0.9798580306327419, + "avg_response_len": 210.83404387763642, + "resp_tokens_sum": 80609445, + "pass_at_k": { + "pass@1": 0.9798580306327419, + "pass@2": 0.9874749653353516, + "pass@4": 0.9917239272631103, + "pass@8": 0.9946187071409932, + "pass@16": 0.9966505338617966, + "pass@32": 0.9979145684044046, + "pass@64": 0.9987759902029831, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9243003731343283, + "pass@2": 0.9516221794570456, + "pass@4": 0.969928064402397, + "pass@8": 0.9822427963159174, + "pass@16": 0.9903780580552475, + "pass@32": 0.9946856168495991, + "pass@64": 0.9965593190653277, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977906341107872, + "pass@2": 0.999128377447717, + "pass@4": 0.9997770731464691, + "pass@8": 0.9999849261956071, + "pass@16": 0.9999999540998669, + "pass@32": 0.9999999999999499, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997715643274854, + "pass@2": 0.999993524658102, + "pass@4": 0.9999999958886717, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9764476102941176, + "pass@2": 0.9835036475220009, + "pass@4": 0.9881245282758772, + "pass@8": 0.9913386482804817, + "pass@16": 0.9933820767780391, + "pass@32": 0.9953309432715226, + "pass@64": 0.9978114867945029, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978558753943217, + "pass@2": 0.9997667447030477, + "pass@4": 0.9999956540329936, + "pass@8": 0.9999999986380128, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9816801819620253, + "pass@2": 0.988404281496063, + "pass@4": 0.9921361622716464, + "pass@8": 0.9951270735067645, + "pass@16": 0.9973320499388789, + "pass@32": 0.9986409115481101, + "pass@64": 0.9996137362637401, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9680930930930931, + "pass@2": 0.9796111318010925, + "pass@4": 0.986489844737876, + "pass@8": 0.9920364232418373, + "pass@16": 0.9957467693807338, + "pass@32": 0.9969236328878157, + "pass@64": 0.996996938016752, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9714881430868167, + "pass@2": 0.9846685349520213, + "pass@4": 0.9885942989039553, + "pass@8": 0.9904067126300237, + "pass@16": 0.9926945429306113, + "pass@32": 0.9954743269009483, + "pass@64": 0.997950753441724, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.61430625, + "avg_response_len": 371.8365171875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.61430625, + "pass@2": 0.6948337475393591, + "pass@4": 0.7552174947975153, + "pass@8": 0.8010594585084798, + "pass@16": 0.8368815265612256, + "pass@32": 0.8654492234953285, + "pass@64": 0.8884205611152566, + "pass@128": 0.9069 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9056796875, + "pass@2": 0.9354986466535433, + "pass@4": 0.9526242564679408, + "pass@8": 0.963458601161703, + "pass@16": 0.9708057396523738, + "pass@32": 0.9767448683064944, + "pass@64": 0.982404059029227, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8490390625, + "pass@2": 0.8908928395669299, + "pass@4": 0.916850836520435, + "pass@8": 0.9355188116322389, + "pass@16": 0.949685963488858, + "pass@32": 0.9607437010233911, + "pass@64": 0.9693829210678885, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.7430078125, + "pass@2": 0.8095824311023627, + "pass@4": 0.853492531589801, + "pass@8": 0.8862884373943454, + "pass@16": 0.9126553093092649, + "pass@32": 0.9326838652147054, + "pass@64": 0.9461192850243815, + "pass@128": 0.957 + }, + "14": { + "pass@1": 0.727171875, + "pass@2": 0.80162155511811, + "pass@4": 0.8517789091676035, + "pass@8": 0.8855516101690537, + "pass@16": 0.9098631619381126, + "pass@32": 0.928837855872377, + "pass@64": 0.9431126453134272, + "pass@128": 0.952 + }, + "15": { + "pass@1": 0.696546875, + "pass@2": 0.776081200787402, + "pass@4": 0.8295034741282343, + "pass@8": 0.8654025161064112, + "pass@16": 0.8906178657863425, + "pass@32": 0.9096555570125101, + "pass@64": 0.9250709620583322, + "pass@128": 0.941 + }, + "16": { + "pass@1": 0.569640625, + "pass@2": 0.6723626968503936, + "pass@4": 0.7457153708286456, + "pass@8": 0.7986660828512879, + "pass@16": 0.8388295574580343, + "pass@32": 0.8710317707718077, + "pass@64": 0.8966970402289524, + "pass@128": 0.915 + }, + "17": { + "pass@1": 0.504015625, + "pass@2": 0.6076593257874024, + "pass@4": 0.6870120757405322, + "pass@8": 0.7481192722915083, + "pass@16": 0.7961898299568897, + "pass@32": 0.8330766882299875, + "pass@64": 0.861536655648687, + "pass@128": 0.884 + }, + "18": { + "pass@1": 0.4738203125, + "pass@2": 0.5751290600393701, + "pass@4": 0.6541842327521556, + "pass@8": 0.7167984534266477, + "pass@16": 0.7665256034868948, + "pass@32": 0.8056501336820523, + "pass@64": 0.8376790693853015, + "pass@128": 0.863 + }, + "19": { + "pass@1": 0.3317578125, + "pass@2": 0.4308519931102363, + "pass@4": 0.5191789361642287, + "pass@8": 0.593997947251625, + "pass@16": 0.6572536679344253, + "pass@32": 0.7122296680249629, + "pass@64": 0.7609197579395308, + "pass@128": 0.802 + }, + "20": { + "pass@1": 0.3423828125, + "pass@2": 0.44865772637795204, + "pass@4": 0.5418343246156725, + "pass@8": 0.6167928528000459, + "pass@16": 0.6763885666009327, + "pass@32": 0.7238381268149598, + "pass@64": 0.7612832154567966, + "pass@128": 0.791 + } + }, + "per_op_accuracy": { + "11": 0.9056796875, + "12": 0.8490390625, + "13": 0.7430078125, + "14": 0.727171875, + "15": 0.696546875, + "16": 0.569640625, + "17": 0.504015625, + "18": 0.4738203125, + "19": 0.3317578125, + "20": 0.3423828125 + }, + "per_op_avg_response_len": { + "11": 297.6541015625, + "12": 307.33053125, + "13": 326.3489765625, + "14": 351.7571484375, + "15": 373.0377734375, + "16": 379.1641328125, + "17": 402.31475, + "18": 417.1555, + "19": 420.4152890625, + "20": 443.18696875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.6541015625, + "12": 307.33053125, + "13": 326.3489765625, + "14": 351.7571484375, + "15": 373.0377734375, + "16": 379.1641328125, + "17": 402.31475, + "18": 417.1555, + "19": 420.4152890625, + "20": 443.18696875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 265633, + "answer_accuracy": 0.6167185178306092, + "avg_response_len": 352.85696972511147, + "resp_tokens_sum": 151982554, + "pass_at_k": { + "pass@1": 0.6167185178306092, + "pass@2": 0.6964144271156341, + "pass@4": 0.7547148796222192, + "pass@8": 0.7988067688407098, + "pass@16": 0.8340209032356112, + "pass@32": 0.8625417044057181, + "pass@64": 0.8853152365021925, + "pass@128": 0.9037147102526003 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8917758498583569, + "pass@2": 0.9224974766344721, + "pass@4": 0.939680541260246, + "pass@8": 0.9510023708562484, + "pass@16": 0.9590901825135237, + "pass@32": 0.9656970528339249, + "pass@64": 0.972682186984747, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8429555084745762, + "pass@2": 0.8858246863739494, + "pass@4": 0.9133644210787202, + "pass@8": 0.9346190779093424, + "pass@16": 0.9514812572300017, + "pass@32": 0.9632039671186756, + "pass@64": 0.9709016775782953, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.757126763322884, + "pass@2": 0.8231551446449289, + "pass@4": 0.8639564725255737, + "pass@8": 0.8946899875422781, + "pass@16": 0.9223208613955524, + "pass@32": 0.9457174763958411, + "pass@64": 0.9605988382503592, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.7394616433021807, + "pass@2": 0.8106104508548578, + "pass@4": 0.8551571903395254, + "pass@8": 0.885358819524307, + "pass@16": 0.9088239976309888, + "pass@32": 0.9283739817552346, + "pass@64": 0.9433370336225584, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.6967124277456648, + "pass@2": 0.7759717354694824, + "pass@4": 0.8292435217635361, + "pass@8": 0.865741190289982, + "pass@16": 0.8937599138215553, + "pass@32": 0.91573542151416, + "pass@64": 0.9309956259048957, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.577898185483871, + "pass@2": 0.6793251524003042, + "pass@4": 0.7511937996661703, + "pass@8": 0.8028934513778322, + "pass@16": 0.8403575221850956, + "pass@32": 0.8677512213969356, + "pass@64": 0.8890210893848132, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.499683887283237, + "pass@2": 0.60846399799736, + "pass@4": 0.6917715551993857, + "pass@8": 0.751368822328008, + "pass@16": 0.794436209124169, + "pass@32": 0.8281025440917178, + "pass@64": 0.8571650620044063, + "pass@128": 0.8815028901734104 + }, + "18": { + "pass@1": 0.4471428571428571, + "pass@2": 0.5437499999999998, + "pass@4": 0.6173458152552356, + "pass@8": 0.6776120314635289, + "pass@16": 0.7313852017826808, + "pass@32": 0.7774102529335, + "pass@64": 0.8135291934065825, + "pass@128": 0.8371428571428572 + }, + "19": { + "pass@1": 0.33877627627627627, + "pass@2": 0.435769430848171, + "pass@4": 0.519100590466732, + "pass@8": 0.5905477300744811, + "pass@16": 0.6522626046447243, + "pass@32": 0.7057474665380328, + "pass@64": 0.7535948299981797, + "pass@128": 0.7987987987987988 + }, + "20": { + "pass@1": 0.36261261261261263, + "pass@2": 0.46963745241304333, + "pass@4": 0.559750553984806, + "pass@8": 0.6293869673980081, + "pass@16": 0.6826706411085892, + "pass@32": 0.7249769927465036, + "pass@64": 0.7591745224001236, + "pass@128": 0.7897897897897898 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 265101, + "answer_accuracy": 0.6158494090098127, + "avg_response_len": 367.4689428151948, + "resp_tokens_sum": 158182151, + "pass_at_k": { + "pass@1": 0.6158494090098127, + "pass@2": 0.6968100051275963, + "pass@4": 0.7587718783144995, + "pass@8": 0.8059544797031246, + "pass@16": 0.8420597759269215, + "pass@32": 0.870412500222662, + "pass@64": 0.8933492555660828, + "pass@128": 0.911685994647636 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9183687876506024, + "pass@2": 0.9470957155393229, + "pass@4": 0.9644122632261325, + "pass@8": 0.9748228712453999, + "pass@16": 0.9804260772125516, + "pass@32": 0.9840721251643764, + "pass@64": 0.9871809068657704, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8847692407375407, + "pass@4": 0.90859120646753, + "pass@8": 0.9243543892077899, + "pass@16": 0.9355822076799643, + "pass@32": 0.9455758004703267, + "pass@64": 0.954258453576493, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7349439775910365, + "pass@2": 0.7977441468713469, + "pass@4": 0.8401459846405754, + "pass@8": 0.8731156714350516, + "pass@16": 0.9001027670028802, + "pass@32": 0.9206109722325958, + "pass@64": 0.9345763304595933, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7219480994152047, + "pass@2": 0.8016178281991068, + "pass@4": 0.8578905791929518, + "pass@8": 0.8960752123638371, + "pass@16": 0.9227277569258882, + "pass@32": 0.9437632815301882, + "pass@64": 0.9595824106465443, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6963304924242424, + "pass@2": 0.7801785075161057, + "pass@4": 0.8355562375725756, + "pass@8": 0.8711732529664689, + "pass@16": 0.8952970708219666, + "pass@32": 0.9137875145220419, + "pass@64": 0.9289625235860818, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5521055911680912, + "pass@2": 0.652413941045831, + "pass@4": 0.7285643666870698, + "pass@8": 0.786036821794172, + "pass@16": 0.8300475577737717, + "pass@32": 0.8653470762432315, + "pass@64": 0.8944466967201555, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5292604813664596, + "pass@2": 0.6346625574656428, + "pass@4": 0.7116403593960688, + "pass@8": 0.7689158035895457, + "pass@16": 0.8124399006328004, + "pass@32": 0.8450682195642002, + "pass@64": 0.8700847005397524, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.4950597426470588, + "pass@2": 0.6015469401343212, + "pass@4": 0.6901890607424076, + "pass@8": 0.7598831637975443, + "pass@16": 0.8089006500258844, + "pass@32": 0.8428722054572504, + "pass@64": 0.872641983259902, + "pass@128": 0.9 + }, + "19": { + "pass@1": 0.32798549107142855, + "pass@2": 0.41885706669478817, + "pass@4": 0.5030038840011073, + "pass@8": 0.5792448263407606, + "pass@16": 0.6469856414243387, + "pass@32": 0.7057171493569743, + "pass@64": 0.7571373760947852, + "pass@128": 0.7976190476190477 + }, + "20": { + "pass@1": 0.3432821856287425, + "pass@2": 0.45409414929982517, + "pass@4": 0.5512342317614491, + "pass@8": 0.628117157107503, + "pass@16": 0.6891362071499393, + "pass@32": 0.737500810422528, + "pass@64": 0.7742154931168451, + "pass@128": 0.8023952095808383 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 255578, + "answer_accuracy": 0.6102393413814181, + "avg_response_len": 395.84456420003056, + "resp_tokens_sum": 165786037, + "pass_at_k": { + "pass@1": 0.6102393413814181, + "pass@2": 0.6911769195921529, + "pass@4": 0.7520811587025832, + "pass@8": 0.7983450161046167, + "pass@16": 0.8345011918649501, + "pass@32": 0.8633380688811664, + "pass@64": 0.8865483783171967, + "pass@128": 0.9052567237163814 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9078869047619048, + "pass@2": 0.9378452693413322, + "pass@4": 0.9547052508317414, + "pass@8": 0.9654399079237302, + "pass@16": 0.9737950717159778, + "pass@32": 0.9814027400049091, + "pass@64": 0.988264082362461, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8589927370030581, + "pass@2": 0.9023532501384576, + "pass@4": 0.9286826807199551, + "pass@8": 0.9473841220031984, + "pass@16": 0.9615011442799076, + "pass@32": 0.9728771141080906, + "pass@64": 0.9824932125818667, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7379918981481481, + "pass@2": 0.8092632084183922, + "pass@4": 0.8578959886958579, + "pass@8": 0.8925309773643367, + "pass@16": 0.9169699589631387, + "pass@32": 0.9331539294981345, + "pass@64": 0.9445818075569187, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7207668768545994, + "pass@2": 0.7930632140937867, + "pass@4": 0.8423586735448866, + "pass@8": 0.8750555088229043, + "pass@16": 0.8977975247178405, + "pass@32": 0.9141328351502144, + "pass@64": 0.9261847865859567, + "pass@128": 0.9347181008902077 + }, + "15": { + "pass@1": 0.6965904706790124, + "pass@2": 0.7720249161563142, + "pass@4": 0.8236162259231482, + "pass@8": 0.8591632431701934, + "pass@16": 0.8824966118297386, + "pass@32": 0.8989543869640029, + "pass@64": 0.9147803787402191, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5802452064896755, + "pass@2": 0.6866507560448752, + "pass@4": 0.7584637174777925, + "pass@8": 0.8078766621663889, + "pass@16": 0.8465251705075594, + "pass@32": 0.879917605832987, + "pass@64": 0.9060463479967112, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.4840455572289157, + "pass@2": 0.5806308402902947, + "pass@4": 0.6581654274578627, + "pass@8": 0.7245625632836872, + "pass@16": 0.7822567879402613, + "pass@32": 0.8266302448029551, + "pass@64": 0.8578020199438595, + "pass@128": 0.8825301204819277 + }, + "18": { + "pass@1": 0.4806451612903226, + "pass@2": 0.5815825819151639, + "pass@4": 0.656286828259371, + "pass@8": 0.7137869894298322, + "pass@16": 0.7597243930779246, + "pass@32": 0.7967096622576195, + "pass@64": 0.826598636724612, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.32852624622356497, + "pass@2": 0.4380809614149439, + "pass@4": 0.535677143548144, + "pass@8": 0.6124449893846747, + "pass@16": 0.6726980213569644, + "pass@32": 0.7253619319753905, + "pass@64": 0.7721284627863696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.32125093843843844, + "pass@2": 0.4222252518266302, + "pass@4": 0.5144899601513776, + "pass@8": 0.5928404269747848, + "pass@16": 0.6573205703414213, + "pass@32": 0.7089955481959461, + "pass@64": 0.7504207951847711, + "pass@128": 0.7807807807807807 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7871159539473684, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7871159539473684, + "pass@2": 0.8331336445814117, + "pass@4": 0.8669764871627748, + "pass@8": 0.8924611626692964, + "pass@16": 0.9122508546551608, + "pass@32": 0.9279394928791536, + "pass@64": 0.9404899826700137, + "pass@128": 0.9504736842105264 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.923390625, + "pass@2": 0.9501561269685036, + "pass@4": 0.9678725301837272, + "pass@8": 0.9793842899999533, + "pass@16": 0.9867344327859435, + "pass@32": 0.9911755389493582, + "pass@64": 0.9936929318542894, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9941796875, + "pass@2": 0.9963764763779527, + "pass@4": 0.9981032145669293, + "pass@8": 0.9992212404213301, + "pass@16": 0.9997815831650843, + "pass@32": 0.9999734313592589, + "pass@64": 0.9999998615770528, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9948125, + "pass@2": 0.9956371801181103, + "pass@4": 0.9962996703224597, + "pass@8": 0.9969636643709774, + "pass@16": 0.9975697588774716, + "pass@32": 0.998123944796517, + "pass@64": 0.9986924084990753, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.975609375, + "pass@2": 0.982559916338583, + "pass@4": 0.9875855872703415, + "pass@8": 0.9912197676299933, + "pass@16": 0.9941905164201755, + "pass@32": 0.9969005728810001, + "pass@64": 0.999023905599881, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992828125, + "pass@2": 0.9953348917322836, + "pass@4": 0.9967678327709036, + "pass@8": 0.9977780583091089, + "pass@16": 0.9985466233300656, + "pass@32": 0.9989439643112072, + "pass@64": 0.9989996702871723, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9856484375, + "pass@2": 0.9913272637795276, + "pass@4": 0.9937962862767151, + "pass@8": 0.9956409850917508, + "pass@16": 0.9971005287140816, + "pass@32": 0.9982284708545968, + "pass@64": 0.9992420693192129, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9763125, + "pass@2": 0.9866906988188983, + "pass@4": 0.9916709403824518, + "pass@8": 0.9949062794665869, + "pass@16": 0.9968789403578321, + "pass@32": 0.9977548704817156, + "pass@64": 0.997985488925269, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.969359375, + "pass@2": 0.9831192175196861, + "pass@4": 0.988282246344207, + "pass@8": 0.9910532203418113, + "pass@16": 0.9931485891853677, + "pass@32": 0.9952573361168118, + "pass@64": 0.9974677235157687, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9056796875, + "pass@2": 0.9354986466535433, + "pass@4": 0.9526242564679408, + "pass@8": 0.963458601161703, + "pass@16": 0.9708057396523738, + "pass@32": 0.9767448683064944, + "pass@64": 0.982404059029227, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8490390625, + "pass@2": 0.8908928395669299, + "pass@4": 0.916850836520435, + "pass@8": 0.9355188116322389, + "pass@16": 0.949685963488858, + "pass@32": 0.9607437010233911, + "pass@64": 0.9693829210678885, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.7430078125, + "pass@2": 0.8095824311023627, + "pass@4": 0.853492531589801, + "pass@8": 0.8862884373943454, + "pass@16": 0.9126553093092649, + "pass@32": 0.9326838652147054, + "pass@64": 0.9461192850243815, + "pass@128": 0.957 + }, + "14": { + "pass@1": 0.727171875, + "pass@2": 0.80162155511811, + "pass@4": 0.8517789091676035, + "pass@8": 0.8855516101690537, + "pass@16": 0.9098631619381126, + "pass@32": 0.928837855872377, + "pass@64": 0.9431126453134272, + "pass@128": 0.952 + }, + "15": { + "pass@1": 0.696546875, + "pass@2": 0.776081200787402, + "pass@4": 0.8295034741282343, + "pass@8": 0.8654025161064112, + "pass@16": 0.8906178657863425, + "pass@32": 0.9096555570125101, + "pass@64": 0.9250709620583322, + "pass@128": 0.941 + }, + "16": { + "pass@1": 0.569640625, + "pass@2": 0.6723626968503936, + "pass@4": 0.7457153708286456, + "pass@8": 0.7986660828512879, + "pass@16": 0.8388295574580343, + "pass@32": 0.8710317707718077, + "pass@64": 0.8966970402289524, + "pass@128": 0.915 + }, + "17": { + "pass@1": 0.504015625, + "pass@2": 0.6076593257874024, + "pass@4": 0.6870120757405322, + "pass@8": 0.7481192722915083, + "pass@16": 0.7961898299568897, + "pass@32": 0.8330766882299875, + "pass@64": 0.861536655648687, + "pass@128": 0.884 + }, + "18": { + "pass@1": 0.4738203125, + "pass@2": 0.5751290600393701, + "pass@4": 0.6541842327521556, + "pass@8": 0.7167984534266477, + "pass@16": 0.7665256034868948, + "pass@32": 0.8056501336820523, + "pass@64": 0.8376790693853015, + "pass@128": 0.863 + }, + "19": { + "pass@1": 0.3317578125, + "pass@2": 0.4308519931102363, + "pass@4": 0.5191789361642287, + "pass@8": 0.593997947251625, + "pass@16": 0.6572536679344253, + "pass@32": 0.7122296680249629, + "pass@64": 0.7609197579395308, + "pass@128": 0.802 + }, + "20": { + "pass@1": 0.3423828125, + "pass@2": 0.44865772637795204, + "pass@4": 0.5418343246156725, + "pass@8": 0.6167928528000459, + "pass@16": 0.6763885666009327, + "pass@32": 0.7238381268149598, + "pass@64": 0.7612832154567966, + "pass@128": 0.791 + } + }, + "per_op_accuracy": { + "10": 0.923390625, + "2": 1.0, + "3": 0.9941796875, + "4": 0.9948125, + "5": 0.975609375, + "6": 0.992828125, + "7": 0.9856484375, + "8": 0.9763125, + "9": 0.969359375, + "11": 0.9056796875, + "12": 0.8490390625, + "13": 0.7430078125, + "14": 0.727171875, + "15": 0.696546875, + "16": 0.569640625, + "17": 0.504015625, + "18": 0.4738203125, + "19": 0.3317578125, + "20": 0.3423828125 + }, + "per_op_avg_response_len": { + "10": 279.618734375, + "2": 115.0162734375, + "3": 144.9221328125, + "4": 159.417390625, + "5": 174.09465625, + "6": 198.336203125, + "7": 223.039484375, + "8": 243.111015625, + "9": 258.7139609375, + "11": 297.6541015625, + "12": 307.33053125, + "13": 326.3489765625, + "14": 351.7571484375, + "15": 373.0377734375, + "16": 379.1641328125, + "17": 402.31475, + "18": 417.1555, + "19": 420.4152890625, + "20": 443.18696875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.618734375, + "2": 115.0162734375, + "3": 144.9221328125, + "4": 159.417390625, + "5": 174.09465625, + "6": 198.336203125, + "7": 223.039484375, + "8": 243.111015625, + "9": 258.7139609375, + "11": 297.6541015625, + "12": 307.33053125, + "13": 326.3489765625, + "14": 351.7571484375, + "15": 373.0377734375, + "16": 379.1641328125, + "17": 402.31475, + "18": 417.1555, + "19": 420.4152890625, + "20": 443.18696875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 636163, + "answer_accuracy": 0.785650243044578, + "avg_response_len": 276.78613312124565, + "resp_tokens_sum": 224121482, + "pass_at_k": { + "pass@1": 0.785650243044578, + "pass@2": 0.8319744116270493, + "pass@4": 0.8651824327167729, + "pass@8": 0.8900061945099712, + "pass@16": 0.9096245615352967, + "pass@32": 0.9254800811760605, + "pass@64": 0.9380569740853564, + "pass@128": 0.9479924122668353 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924940081120944, + "pass@2": 0.9509071667479613, + "pass@4": 0.96726450316498, + "pass@8": 0.9770110038469436, + "pass@16": 0.9828349449512709, + "pass@32": 0.9871731776191788, + "pass@64": 0.9900107432528562, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942660550458715, + "pass@2": 0.9962657871607791, + "pass@4": 0.9981546572733454, + "pass@8": 0.9994911036229956, + "pass@16": 0.9999636877139981, + "pass@32": 0.9999998965218795, + "pass@64": 0.9999999999999889, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9873779296875, + "pass@2": 0.9894150621309056, + "pass@4": 0.9913685054016685, + "pass@8": 0.99325659141523, + "pass@16": 0.9947953833031221, + "pass@32": 0.9958905263080137, + "pass@64": 0.9966888749848067, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.966468253968254, + "pass@2": 0.9759350393700784, + "pass@4": 0.9829707724034493, + "pass@8": 0.9879597908973488, + "pass@16": 0.9924307677634848, + "pass@32": 0.9968554804125532, + "pass@64": 0.9996607862988176, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9938014595808383, + "pass@2": 0.9963425756754208, + "pass@4": 0.997830821203487, + "pass@8": 0.9988929507572979, + "pass@16": 0.9996441380031399, + "pass@32": 0.9999703661222745, + "pass@64": 0.9999999638637934, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842825443786982, + "pass@2": 0.9909884889111497, + "pass@4": 0.9934844918608546, + "pass@8": 0.9952204293474521, + "pass@16": 0.9966938224900305, + "pass@32": 0.9982559181300668, + "pass@64": 0.9995980149616418, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9789162660256411, + "pass@2": 0.9889677815213003, + "pass@4": 0.9928349941834194, + "pass@8": 0.9948939475896649, + "pass@16": 0.9959889984825363, + "pass@32": 0.9966354953643705, + "pass@64": 0.9967927316172044, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9708406690140845, + "pass@2": 0.9864121243207277, + "pass@4": 0.9930078146041601, + "pass@8": 0.9962900913419857, + "pass@16": 0.9976705029288833, + "pass@32": 0.998414543391346, + "pass@64": 0.9993013192675217, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8917758498583569, + "pass@2": 0.9224974766344721, + "pass@4": 0.939680541260246, + "pass@8": 0.9510023708562484, + "pass@16": 0.9590901825135237, + "pass@32": 0.9656970528339249, + "pass@64": 0.972682186984747, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8429555084745762, + "pass@2": 0.8858246863739494, + "pass@4": 0.9133644210787202, + "pass@8": 0.9346190779093424, + "pass@16": 0.9514812572300017, + "pass@32": 0.9632039671186756, + "pass@64": 0.9709016775782953, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.757126763322884, + "pass@2": 0.8231551446449289, + "pass@4": 0.8639564725255737, + "pass@8": 0.8946899875422781, + "pass@16": 0.9223208613955524, + "pass@32": 0.9457174763958411, + "pass@64": 0.9605988382503592, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.7394616433021807, + "pass@2": 0.8106104508548578, + "pass@4": 0.8551571903395254, + "pass@8": 0.885358819524307, + "pass@16": 0.9088239976309888, + "pass@32": 0.9283739817552346, + "pass@64": 0.9433370336225584, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.6967124277456648, + "pass@2": 0.7759717354694824, + "pass@4": 0.8292435217635361, + "pass@8": 0.865741190289982, + "pass@16": 0.8937599138215553, + "pass@32": 0.91573542151416, + "pass@64": 0.9309956259048957, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.577898185483871, + "pass@2": 0.6793251524003042, + "pass@4": 0.7511937996661703, + "pass@8": 0.8028934513778322, + "pass@16": 0.8403575221850956, + "pass@32": 0.8677512213969356, + "pass@64": 0.8890210893848132, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.499683887283237, + "pass@2": 0.60846399799736, + "pass@4": 0.6917715551993857, + "pass@8": 0.751368822328008, + "pass@16": 0.794436209124169, + "pass@32": 0.8281025440917178, + "pass@64": 0.8571650620044063, + "pass@128": 0.8815028901734104 + }, + "18": { + "pass@1": 0.4471428571428571, + "pass@2": 0.5437499999999998, + "pass@4": 0.6173458152552356, + "pass@8": 0.6776120314635289, + "pass@16": 0.7313852017826808, + "pass@32": 0.7774102529335, + "pass@64": 0.8135291934065825, + "pass@128": 0.8371428571428572 + }, + "19": { + "pass@1": 0.33877627627627627, + "pass@2": 0.435769430848171, + "pass@4": 0.519100590466732, + "pass@8": 0.5905477300744811, + "pass@16": 0.6522626046447243, + "pass@32": 0.7057474665380328, + "pass@64": 0.7535948299981797, + "pass@128": 0.7987987987987988 + }, + "20": { + "pass@1": 0.36261261261261263, + "pass@2": 0.46963745241304333, + "pass@4": 0.559750553984806, + "pass@8": 0.6293869673980081, + "pass@16": 0.6826706411085892, + "pass@32": 0.7249769927465036, + "pass@64": 0.7591745224001236, + "pass@128": 0.7897897897897898 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 647890, + "answer_accuracy": 0.7890320537802027, + "avg_response_len": 286.6284087587685, + "resp_tokens_sum": 235356319, + "pass_at_k": { + "pass@1": 0.7890320537802027, + "pass@2": 0.8348169160923194, + "pass@4": 0.8692627068927307, + "pass@8": 0.8953190854749478, + "pass@16": 0.915198475097219, + "pass@32": 0.9307327192487689, + "pass@64": 0.9432628333711802, + "pass@128": 0.9532346063912704 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9208445168711656, + "pass@2": 0.9478686114197385, + "pass@4": 0.9663925215828081, + "pass@8": 0.978914794263826, + "pass@16": 0.987045205518265, + "pass@32": 0.9917305217541144, + "pass@64": 0.9945764355971969, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9903409090909091, + "pass@2": 0.9936258500357912, + "pass@4": 0.9963124410585041, + "pass@8": 0.9981600601561137, + "pass@16": 0.9993741606859169, + "pass@32": 0.999919591504913, + "pass@64": 0.9999995805365348, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968333949704142, + "pass@2": 0.9971200437963007, + "pass@4": 0.9972241124260354, + "pass@8": 0.9974010506452778, + "pass@16": 0.9977373852676699, + "pass@32": 0.9983401667986769, + "pass@64": 0.9992661790057308, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9831295289855072, + "pass@2": 0.9876786631290653, + "pass@4": 0.9912679836216123, + "pass@8": 0.994079110382506, + "pass@16": 0.9965939665801291, + "pass@32": 0.9984886256195019, + "pass@64": 0.9996372533496305, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9873298710601719, + "pass@2": 0.9903450240281569, + "pass@4": 0.992818670946934, + "pass@8": 0.9946928744639584, + "pass@16": 0.9961761640029136, + "pass@32": 0.9970024699895921, + "pass@64": 0.9971337603342841, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.990606936416185, + "pass@2": 0.9943277502161942, + "pass@4": 0.9956170541904806, + "pass@8": 0.9965211697808511, + "pass@16": 0.9972863841958547, + "pass@32": 0.997824978258472, + "pass@64": 0.9985549121469247, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9817341549295775, + "pass@2": 0.9913302650548964, + "pass@4": 0.9955078983084861, + "pass@8": 0.9976091236030408, + "pass@16": 0.9987230947253419, + "pass@32": 0.9995183835954631, + "pass@64": 0.9999610600031621, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658027694610778, + "pass@2": 0.9781766437597247, + "pass@4": 0.9829689976065864, + "pass@8": 0.9860890727169717, + "pass@16": 0.9887651430963902, + "pass@32": 0.9916995734930811, + "pass@64": 0.9950690744174322, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9183687876506024, + "pass@2": 0.9470957155393229, + "pass@4": 0.9644122632261325, + "pass@8": 0.9748228712453999, + "pass@16": 0.9804260772125516, + "pass@32": 0.9840721251643764, + "pass@64": 0.9871809068657704, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8847692407375407, + "pass@4": 0.90859120646753, + "pass@8": 0.9243543892077899, + "pass@16": 0.9355822076799643, + "pass@32": 0.9455758004703267, + "pass@64": 0.954258453576493, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7349439775910365, + "pass@2": 0.7977441468713469, + "pass@4": 0.8401459846405754, + "pass@8": 0.8731156714350516, + "pass@16": 0.9001027670028802, + "pass@32": 0.9206109722325958, + "pass@64": 0.9345763304595933, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7219480994152047, + "pass@2": 0.8016178281991068, + "pass@4": 0.8578905791929518, + "pass@8": 0.8960752123638371, + "pass@16": 0.9227277569258882, + "pass@32": 0.9437632815301882, + "pass@64": 0.9595824106465443, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6963304924242424, + "pass@2": 0.7801785075161057, + "pass@4": 0.8355562375725756, + "pass@8": 0.8711732529664689, + "pass@16": 0.8952970708219666, + "pass@32": 0.9137875145220419, + "pass@64": 0.9289625235860818, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5521055911680912, + "pass@2": 0.652413941045831, + "pass@4": 0.7285643666870698, + "pass@8": 0.786036821794172, + "pass@16": 0.8300475577737717, + "pass@32": 0.8653470762432315, + "pass@64": 0.8944466967201555, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5292604813664596, + "pass@2": 0.6346625574656428, + "pass@4": 0.7116403593960688, + "pass@8": 0.7689158035895457, + "pass@16": 0.8124399006328004, + "pass@32": 0.8450682195642002, + "pass@64": 0.8700847005397524, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.4950597426470588, + "pass@2": 0.6015469401343212, + "pass@4": 0.6901890607424076, + "pass@8": 0.7598831637975443, + "pass@16": 0.8089006500258844, + "pass@32": 0.8428722054572504, + "pass@64": 0.872641983259902, + "pass@128": 0.9 + }, + "19": { + "pass@1": 0.32798549107142855, + "pass@2": 0.41885706669478817, + "pass@4": 0.5030038840011073, + "pass@8": 0.5792448263407606, + "pass@16": 0.6469856414243387, + "pass@32": 0.7057171493569743, + "pass@64": 0.7571373760947852, + "pass@128": 0.7976190476190477 + }, + "20": { + "pass@1": 0.3432821856287425, + "pass@2": 0.45409414929982517, + "pass@4": 0.5512342317614491, + "pass@8": 0.628117157107503, + "pass@16": 0.6891362071499393, + "pass@32": 0.737500810422528, + "pass@64": 0.7742154931168451, + "pass@128": 0.8023952095808383 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630213, + "answer_accuracy": 0.786633497763221, + "avg_response_len": 307.55147837114555, + "resp_tokens_sum": 246395482, + "pass_at_k": { + "pass@1": 0.786633497763221, + "pass@2": 0.8325800610899756, + "pass@4": 0.8664465444974782, + "pass@8": 0.8920132562588934, + "pass@16": 0.9118841738979606, + "pass@32": 0.9275623865159199, + "pass@64": 0.9401070740677763, + "pass@128": 0.950151781434734 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9243003731343283, + "pass@2": 0.9516221794570456, + "pass@4": 0.969928064402397, + "pass@8": 0.9822427963159174, + "pass@16": 0.9903780580552475, + "pass@32": 0.9946856168495991, + "pass@64": 0.9965593190653277, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977906341107872, + "pass@2": 0.999128377447717, + "pass@4": 0.9997770731464691, + "pass@8": 0.9999849261956071, + "pass@16": 0.9999999540998669, + "pass@32": 0.9999999999999499, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997715643274854, + "pass@2": 0.999993524658102, + "pass@4": 0.9999999958886717, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9764476102941176, + "pass@2": 0.9835036475220009, + "pass@4": 0.9881245282758772, + "pass@8": 0.9913386482804817, + "pass@16": 0.9933820767780391, + "pass@32": 0.9953309432715226, + "pass@64": 0.9978114867945029, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978558753943217, + "pass@2": 0.9997667447030477, + "pass@4": 0.9999956540329936, + "pass@8": 0.9999999986380128, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9816801819620253, + "pass@2": 0.988404281496063, + "pass@4": 0.9921361622716464, + "pass@8": 0.9951270735067645, + "pass@16": 0.9973320499388789, + "pass@32": 0.9986409115481101, + "pass@64": 0.9996137362637401, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9680930930930931, + "pass@2": 0.9796111318010925, + "pass@4": 0.986489844737876, + "pass@8": 0.9920364232418373, + "pass@16": 0.9957467693807338, + "pass@32": 0.9969236328878157, + "pass@64": 0.996996938016752, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9714881430868167, + "pass@2": 0.9846685349520213, + "pass@4": 0.9885942989039553, + "pass@8": 0.9904067126300237, + "pass@16": 0.9926945429306113, + "pass@32": 0.9954743269009483, + "pass@64": 0.997950753441724, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9078869047619048, + "pass@2": 0.9378452693413322, + "pass@4": 0.9547052508317414, + "pass@8": 0.9654399079237302, + "pass@16": 0.9737950717159778, + "pass@32": 0.9814027400049091, + "pass@64": 0.988264082362461, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8589927370030581, + "pass@2": 0.9023532501384576, + "pass@4": 0.9286826807199551, + "pass@8": 0.9473841220031984, + "pass@16": 0.9615011442799076, + "pass@32": 0.9728771141080906, + "pass@64": 0.9824932125818667, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7379918981481481, + "pass@2": 0.8092632084183922, + "pass@4": 0.8578959886958579, + "pass@8": 0.8925309773643367, + "pass@16": 0.9169699589631387, + "pass@32": 0.9331539294981345, + "pass@64": 0.9445818075569187, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7207668768545994, + "pass@2": 0.7930632140937867, + "pass@4": 0.8423586735448866, + "pass@8": 0.8750555088229043, + "pass@16": 0.8977975247178405, + "pass@32": 0.9141328351502144, + "pass@64": 0.9261847865859567, + "pass@128": 0.9347181008902077 + }, + "15": { + "pass@1": 0.6965904706790124, + "pass@2": 0.7720249161563142, + "pass@4": 0.8236162259231482, + "pass@8": 0.8591632431701934, + "pass@16": 0.8824966118297386, + "pass@32": 0.8989543869640029, + "pass@64": 0.9147803787402191, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5802452064896755, + "pass@2": 0.6866507560448752, + "pass@4": 0.7584637174777925, + "pass@8": 0.8078766621663889, + "pass@16": 0.8465251705075594, + "pass@32": 0.879917605832987, + "pass@64": 0.9060463479967112, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.4840455572289157, + "pass@2": 0.5806308402902947, + "pass@4": 0.6581654274578627, + "pass@8": 0.7245625632836872, + "pass@16": 0.7822567879402613, + "pass@32": 0.8266302448029551, + "pass@64": 0.8578020199438595, + "pass@128": 0.8825301204819277 + }, + "18": { + "pass@1": 0.4806451612903226, + "pass@2": 0.5815825819151639, + "pass@4": 0.656286828259371, + "pass@8": 0.7137869894298322, + "pass@16": 0.7597243930779246, + "pass@32": 0.7967096622576195, + "pass@64": 0.826598636724612, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.32852624622356497, + "pass@2": 0.4380809614149439, + "pass@4": 0.535677143548144, + "pass@8": 0.6124449893846747, + "pass@16": 0.6726980213569644, + "pass@32": 0.7253619319753905, + "pass@64": 0.7721284627863696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.32125093843843844, + "pass@2": 0.4222252518266302, + "pass@4": 0.5144899601513776, + "pass@8": 0.5928404269747848, + "pass@16": 0.6573205703414213, + "pass@32": 0.7089955481959461, + "pass@64": 0.7504207951847711, + "pass@128": 0.7807807807807807 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f35683468515f976a4f8ed2c9d720ab2a77145b4 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bafcf5b0e96038134b18b743af49a9ec6c091f5b85db8d1c54c0f30f19fcdbd4 +size 2706999359 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/summary.csv b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8311c6a628de45c1bdeb3bdf44f129700d445b2d --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op17-20,0.9791267361111111,0.61430625,0.7871159539473684,0.0,0.0,0.0,199.5855390625,371.8365171875,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/summary.json b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..08cd8704be764d116563cc684f8205adbe0925a0 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op17-20", + "id_acc": 0.9791267361111111, + "ood_acc": 0.61430625, + "total_acc": 0.7871159539473684, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.5855390625, + "ood_avg_resp_len": 371.8365171875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 636163, + "answer_accuracy": 0.785650243044578, + "avg_response_len": 276.78613312124565, + "resp_tokens_sum": 224121482, + "pass_at_k": { + "pass@1": 0.785650243044578, + "pass@2": 0.8319744116270493, + "pass@4": 0.8651824327167729, + "pass@8": 0.8900061945099712, + "pass@16": 0.9096245615352967, + "pass@32": 0.9254800811760605, + "pass@64": 0.9380569740853564, + "pass@128": 0.9479924122668353 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924940081120944, + "pass@2": 0.9509071667479613, + "pass@4": 0.96726450316498, + "pass@8": 0.9770110038469436, + "pass@16": 0.9828349449512709, + "pass@32": 0.9871731776191788, + "pass@64": 0.9900107432528562, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942660550458715, + "pass@2": 0.9962657871607791, + "pass@4": 0.9981546572733454, + "pass@8": 0.9994911036229956, + "pass@16": 0.9999636877139981, + "pass@32": 0.9999998965218795, + "pass@64": 0.9999999999999889, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9873779296875, + "pass@2": 0.9894150621309056, + "pass@4": 0.9913685054016685, + "pass@8": 0.99325659141523, + "pass@16": 0.9947953833031221, + "pass@32": 0.9958905263080137, + "pass@64": 0.9966888749848067, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.966468253968254, + "pass@2": 0.9759350393700784, + "pass@4": 0.9829707724034493, + "pass@8": 0.9879597908973488, + "pass@16": 0.9924307677634848, + "pass@32": 0.9968554804125532, + "pass@64": 0.9996607862988176, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9938014595808383, + "pass@2": 0.9963425756754208, + "pass@4": 0.997830821203487, + "pass@8": 0.9988929507572979, + "pass@16": 0.9996441380031399, + "pass@32": 0.9999703661222745, + "pass@64": 0.9999999638637934, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842825443786982, + "pass@2": 0.9909884889111497, + "pass@4": 0.9934844918608546, + "pass@8": 0.9952204293474521, + "pass@16": 0.9966938224900305, + "pass@32": 0.9982559181300668, + "pass@64": 0.9995980149616418, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9789162660256411, + "pass@2": 0.9889677815213003, + "pass@4": 0.9928349941834194, + "pass@8": 0.9948939475896649, + "pass@16": 0.9959889984825363, + "pass@32": 0.9966354953643705, + "pass@64": 0.9967927316172044, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9708406690140845, + "pass@2": 0.9864121243207277, + "pass@4": 0.9930078146041601, + "pass@8": 0.9962900913419857, + "pass@16": 0.9976705029288833, + "pass@32": 0.998414543391346, + "pass@64": 0.9993013192675217, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8917758498583569, + "pass@2": 0.9224974766344721, + "pass@4": 0.939680541260246, + "pass@8": 0.9510023708562484, + "pass@16": 0.9590901825135237, + "pass@32": 0.9656970528339249, + "pass@64": 0.972682186984747, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8429555084745762, + "pass@2": 0.8858246863739494, + "pass@4": 0.9133644210787202, + "pass@8": 0.9346190779093424, + "pass@16": 0.9514812572300017, + "pass@32": 0.9632039671186756, + "pass@64": 0.9709016775782953, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.757126763322884, + "pass@2": 0.8231551446449289, + "pass@4": 0.8639564725255737, + "pass@8": 0.8946899875422781, + "pass@16": 0.9223208613955524, + "pass@32": 0.9457174763958411, + "pass@64": 0.9605988382503592, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.7394616433021807, + "pass@2": 0.8106104508548578, + "pass@4": 0.8551571903395254, + "pass@8": 0.885358819524307, + "pass@16": 0.9088239976309888, + "pass@32": 0.9283739817552346, + "pass@64": 0.9433370336225584, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.6967124277456648, + "pass@2": 0.7759717354694824, + "pass@4": 0.8292435217635361, + "pass@8": 0.865741190289982, + "pass@16": 0.8937599138215553, + "pass@32": 0.91573542151416, + "pass@64": 0.9309956259048957, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.577898185483871, + "pass@2": 0.6793251524003042, + "pass@4": 0.7511937996661703, + "pass@8": 0.8028934513778322, + "pass@16": 0.8403575221850956, + "pass@32": 0.8677512213969356, + "pass@64": 0.8890210893848132, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.499683887283237, + "pass@2": 0.60846399799736, + "pass@4": 0.6917715551993857, + "pass@8": 0.751368822328008, + "pass@16": 0.794436209124169, + "pass@32": 0.8281025440917178, + "pass@64": 0.8571650620044063, + "pass@128": 0.8815028901734104 + }, + "18": { + "pass@1": 0.4471428571428571, + "pass@2": 0.5437499999999998, + "pass@4": 0.6173458152552356, + "pass@8": 0.6776120314635289, + "pass@16": 0.7313852017826808, + "pass@32": 0.7774102529335, + "pass@64": 0.8135291934065825, + "pass@128": 0.8371428571428572 + }, + "19": { + "pass@1": 0.33877627627627627, + "pass@2": 0.435769430848171, + "pass@4": 0.519100590466732, + "pass@8": 0.5905477300744811, + "pass@16": 0.6522626046447243, + "pass@32": 0.7057474665380328, + "pass@64": 0.7535948299981797, + "pass@128": 0.7987987987987988 + }, + "20": { + "pass@1": 0.36261261261261263, + "pass@2": 0.46963745241304333, + "pass@4": 0.559750553984806, + "pass@8": 0.6293869673980081, + "pass@16": 0.6826706411085892, + "pass@32": 0.7249769927465036, + "pass@64": 0.7591745224001236, + "pass@128": 0.7897897897897898 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 647890, + "answer_accuracy": 0.7890320537802027, + "avg_response_len": 286.6284087587685, + "resp_tokens_sum": 235356319, + "pass_at_k": { + "pass@1": 0.7890320537802027, + "pass@2": 0.8348169160923194, + "pass@4": 0.8692627068927307, + "pass@8": 0.8953190854749478, + "pass@16": 0.915198475097219, + "pass@32": 0.9307327192487689, + "pass@64": 0.9432628333711802, + "pass@128": 0.9532346063912704 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9208445168711656, + "pass@2": 0.9478686114197385, + "pass@4": 0.9663925215828081, + "pass@8": 0.978914794263826, + "pass@16": 0.987045205518265, + "pass@32": 0.9917305217541144, + "pass@64": 0.9945764355971969, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9903409090909091, + "pass@2": 0.9936258500357912, + "pass@4": 0.9963124410585041, + "pass@8": 0.9981600601561137, + "pass@16": 0.9993741606859169, + "pass@32": 0.999919591504913, + "pass@64": 0.9999995805365348, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968333949704142, + "pass@2": 0.9971200437963007, + "pass@4": 0.9972241124260354, + "pass@8": 0.9974010506452778, + "pass@16": 0.9977373852676699, + "pass@32": 0.9983401667986769, + "pass@64": 0.9992661790057308, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9831295289855072, + "pass@2": 0.9876786631290653, + "pass@4": 0.9912679836216123, + "pass@8": 0.994079110382506, + "pass@16": 0.9965939665801291, + "pass@32": 0.9984886256195019, + "pass@64": 0.9996372533496305, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9873298710601719, + "pass@2": 0.9903450240281569, + "pass@4": 0.992818670946934, + "pass@8": 0.9946928744639584, + "pass@16": 0.9961761640029136, + "pass@32": 0.9970024699895921, + "pass@64": 0.9971337603342841, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.990606936416185, + "pass@2": 0.9943277502161942, + "pass@4": 0.9956170541904806, + "pass@8": 0.9965211697808511, + "pass@16": 0.9972863841958547, + "pass@32": 0.997824978258472, + "pass@64": 0.9985549121469247, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9817341549295775, + "pass@2": 0.9913302650548964, + "pass@4": 0.9955078983084861, + "pass@8": 0.9976091236030408, + "pass@16": 0.9987230947253419, + "pass@32": 0.9995183835954631, + "pass@64": 0.9999610600031621, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658027694610778, + "pass@2": 0.9781766437597247, + "pass@4": 0.9829689976065864, + "pass@8": 0.9860890727169717, + "pass@16": 0.9887651430963902, + "pass@32": 0.9916995734930811, + "pass@64": 0.9950690744174322, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9183687876506024, + "pass@2": 0.9470957155393229, + "pass@4": 0.9644122632261325, + "pass@8": 0.9748228712453999, + "pass@16": 0.9804260772125516, + "pass@32": 0.9840721251643764, + "pass@64": 0.9871809068657704, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8847692407375407, + "pass@4": 0.90859120646753, + "pass@8": 0.9243543892077899, + "pass@16": 0.9355822076799643, + "pass@32": 0.9455758004703267, + "pass@64": 0.954258453576493, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7349439775910365, + "pass@2": 0.7977441468713469, + "pass@4": 0.8401459846405754, + "pass@8": 0.8731156714350516, + "pass@16": 0.9001027670028802, + "pass@32": 0.9206109722325958, + "pass@64": 0.9345763304595933, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7219480994152047, + "pass@2": 0.8016178281991068, + "pass@4": 0.8578905791929518, + "pass@8": 0.8960752123638371, + "pass@16": 0.9227277569258882, + "pass@32": 0.9437632815301882, + "pass@64": 0.9595824106465443, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6963304924242424, + "pass@2": 0.7801785075161057, + "pass@4": 0.8355562375725756, + "pass@8": 0.8711732529664689, + "pass@16": 0.8952970708219666, + "pass@32": 0.9137875145220419, + "pass@64": 0.9289625235860818, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5521055911680912, + "pass@2": 0.652413941045831, + "pass@4": 0.7285643666870698, + "pass@8": 0.786036821794172, + "pass@16": 0.8300475577737717, + "pass@32": 0.8653470762432315, + "pass@64": 0.8944466967201555, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5292604813664596, + "pass@2": 0.6346625574656428, + "pass@4": 0.7116403593960688, + "pass@8": 0.7689158035895457, + "pass@16": 0.8124399006328004, + "pass@32": 0.8450682195642002, + "pass@64": 0.8700847005397524, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.4950597426470588, + "pass@2": 0.6015469401343212, + "pass@4": 0.6901890607424076, + "pass@8": 0.7598831637975443, + "pass@16": 0.8089006500258844, + "pass@32": 0.8428722054572504, + "pass@64": 0.872641983259902, + "pass@128": 0.9 + }, + "19": { + "pass@1": 0.32798549107142855, + "pass@2": 0.41885706669478817, + "pass@4": 0.5030038840011073, + "pass@8": 0.5792448263407606, + "pass@16": 0.6469856414243387, + "pass@32": 0.7057171493569743, + "pass@64": 0.7571373760947852, + "pass@128": 0.7976190476190477 + }, + "20": { + "pass@1": 0.3432821856287425, + "pass@2": 0.45409414929982517, + "pass@4": 0.5512342317614491, + "pass@8": 0.628117157107503, + "pass@16": 0.6891362071499393, + "pass@32": 0.737500810422528, + "pass@64": 0.7742154931168451, + "pass@128": 0.8023952095808383 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630213, + "answer_accuracy": 0.786633497763221, + "avg_response_len": 307.55147837114555, + "resp_tokens_sum": 246395482, + "pass_at_k": { + "pass@1": 0.786633497763221, + "pass@2": 0.8325800610899756, + "pass@4": 0.8664465444974782, + "pass@8": 0.8920132562588934, + "pass@16": 0.9118841738979606, + "pass@32": 0.9275623865159199, + "pass@64": 0.9401070740677763, + "pass@128": 0.950151781434734 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9243003731343283, + "pass@2": 0.9516221794570456, + "pass@4": 0.969928064402397, + "pass@8": 0.9822427963159174, + "pass@16": 0.9903780580552475, + "pass@32": 0.9946856168495991, + "pass@64": 0.9965593190653277, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977906341107872, + "pass@2": 0.999128377447717, + "pass@4": 0.9997770731464691, + "pass@8": 0.9999849261956071, + "pass@16": 0.9999999540998669, + "pass@32": 0.9999999999999499, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997715643274854, + "pass@2": 0.999993524658102, + "pass@4": 0.9999999958886717, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9764476102941176, + "pass@2": 0.9835036475220009, + "pass@4": 0.9881245282758772, + "pass@8": 0.9913386482804817, + "pass@16": 0.9933820767780391, + "pass@32": 0.9953309432715226, + "pass@64": 0.9978114867945029, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978558753943217, + "pass@2": 0.9997667447030477, + "pass@4": 0.9999956540329936, + "pass@8": 0.9999999986380128, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9816801819620253, + "pass@2": 0.988404281496063, + "pass@4": 0.9921361622716464, + "pass@8": 0.9951270735067645, + "pass@16": 0.9973320499388789, + "pass@32": 0.9986409115481101, + "pass@64": 0.9996137362637401, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9680930930930931, + "pass@2": 0.9796111318010925, + "pass@4": 0.986489844737876, + "pass@8": 0.9920364232418373, + "pass@16": 0.9957467693807338, + "pass@32": 0.9969236328878157, + "pass@64": 0.996996938016752, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9714881430868167, + "pass@2": 0.9846685349520213, + "pass@4": 0.9885942989039553, + "pass@8": 0.9904067126300237, + "pass@16": 0.9926945429306113, + "pass@32": 0.9954743269009483, + "pass@64": 0.997950753441724, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9078869047619048, + "pass@2": 0.9378452693413322, + "pass@4": 0.9547052508317414, + "pass@8": 0.9654399079237302, + "pass@16": 0.9737950717159778, + "pass@32": 0.9814027400049091, + "pass@64": 0.988264082362461, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8589927370030581, + "pass@2": 0.9023532501384576, + "pass@4": 0.9286826807199551, + "pass@8": 0.9473841220031984, + "pass@16": 0.9615011442799076, + "pass@32": 0.9728771141080906, + "pass@64": 0.9824932125818667, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7379918981481481, + "pass@2": 0.8092632084183922, + "pass@4": 0.8578959886958579, + "pass@8": 0.8925309773643367, + "pass@16": 0.9169699589631387, + "pass@32": 0.9331539294981345, + "pass@64": 0.9445818075569187, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7207668768545994, + "pass@2": 0.7930632140937867, + "pass@4": 0.8423586735448866, + "pass@8": 0.8750555088229043, + "pass@16": 0.8977975247178405, + "pass@32": 0.9141328351502144, + "pass@64": 0.9261847865859567, + "pass@128": 0.9347181008902077 + }, + "15": { + "pass@1": 0.6965904706790124, + "pass@2": 0.7720249161563142, + "pass@4": 0.8236162259231482, + "pass@8": 0.8591632431701934, + "pass@16": 0.8824966118297386, + "pass@32": 0.8989543869640029, + "pass@64": 0.9147803787402191, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5802452064896755, + "pass@2": 0.6866507560448752, + "pass@4": 0.7584637174777925, + "pass@8": 0.8078766621663889, + "pass@16": 0.8465251705075594, + "pass@32": 0.879917605832987, + "pass@64": 0.9060463479967112, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.4840455572289157, + "pass@2": 0.5806308402902947, + "pass@4": 0.6581654274578627, + "pass@8": 0.7245625632836872, + "pass@16": 0.7822567879402613, + "pass@32": 0.8266302448029551, + "pass@64": 0.8578020199438595, + "pass@128": 0.8825301204819277 + }, + "18": { + "pass@1": 0.4806451612903226, + "pass@2": 0.5815825819151639, + "pass@4": 0.656286828259371, + "pass@8": 0.7137869894298322, + "pass@16": 0.7597243930779246, + "pass@32": 0.7967096622576195, + "pass@64": 0.826598636724612, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.32852624622356497, + "pass@2": 0.4380809614149439, + "pass@4": 0.535677143548144, + "pass@8": 0.6124449893846747, + "pass@16": 0.6726980213569644, + "pass@32": 0.7253619319753905, + "pass@64": 0.7721284627863696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.32125093843843844, + "pass@2": 0.4222252518266302, + "pass@4": 0.5144899601513776, + "pass@8": 0.5928404269747848, + "pass@16": 0.6573205703414213, + "pass@32": 0.7089955481959461, + "pass@64": 0.7504207951847711, + "pass@128": 0.7807807807807807 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d14ba8b9067d2f1f86081d17e6a80d5fca7b0c4d --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:730fbb917c657e6d00c70326b1d2657615e3e9000aa668b2675271a10f88abfb +size 1789734352 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6f7965ad7c7e0250e9de815003aa6314d72c49bc --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9850538194444445, + "avg_response_len": 199.6154670138889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9850538194444445, + "pass@2": 0.9890675853018349, + "pass@4": 0.9915910924259462, + "pass@8": 0.9935450217299747, + "pass@16": 0.9951385359115983, + "pass@32": 0.9964060909375977, + "pass@64": 0.9973418446158011, + "pass@128": 0.9978888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9334296875, + "pass@2": 0.9509089566929134, + "pass@4": 0.962810891544807, + "pass@8": 0.971979454952471, + "pass@16": 0.9791457199182901, + "pass@32": 0.984637008012194, + "pass@64": 0.9891191210637711, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982734375, + "pass@2": 0.9991943897637795, + "pass@4": 0.9997140119047618, + "pass@8": 0.999939962982416, + "pass@16": 0.9999972095036287, + "pass@32": 0.9999999974929434, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996796875, + "pass@2": 0.9978724163385826, + "pass@4": 0.9986181589801275, + "pass@8": 0.998986334898379, + "pass@16": 0.9991220253445506, + "pass@32": 0.9992499974876511, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9795234375, + "pass@2": 0.9837289616141732, + "pass@4": 0.987008391263592, + "pass@8": 0.9898293216761134, + "pass@16": 0.99239701401116, + "pass@32": 0.9945059926298492, + "pass@64": 0.995966407010378, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9971015625, + "pass@2": 0.9980504429133858, + "pass@4": 0.9986530571803525, + "pass@8": 0.9991347816536281, + "pass@16": 0.9994783650002377, + "pass@32": 0.9997688683527692, + "pass@64": 0.9999711811022796, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9927734375, + "pass@2": 0.9944325787401576, + "pass@4": 0.99508110808024, + "pass@8": 0.9956768556165126, + "pass@16": 0.996358401907696, + "pass@32": 0.9971439391473108, + "pass@64": 0.9978118217783716, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.987078125, + "pass@2": 0.9907172736220474, + "pass@4": 0.9926614599737532, + "pass@8": 0.9941317677465087, + "pass@16": 0.9953316946977685, + "pass@32": 0.9962961596053821, + "pass@64": 0.9968609707442045, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9805078125, + "pass@2": 0.9867032480314963, + "pass@4": 0.9897727529058868, + "pass@8": 0.9922267160437407, + "pass@16": 0.9944163928210517, + "pass@32": 0.996052855710281, + "pass@64": 0.9968470998432064, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9334296875, + "2": 1.0, + "3": 0.9982734375, + "4": 0.996796875, + "5": 0.9795234375, + "6": 0.9971015625, + "7": 0.9927734375, + "8": 0.987078125, + "9": 0.9805078125 + }, + "per_op_avg_response_len": { + "10": 279.63134375, + "2": 115.39703125, + "3": 144.4703671875, + "4": 159.7310078125, + "5": 173.3921328125, + "6": 198.5842890625, + "7": 223.635484375, + "8": 242.8098359375, + "9": 258.8877109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.63134375, + "2": 115.39703125, + "3": 144.4703671875, + "4": 159.7310078125, + "5": 173.3921328125, + "6": 198.5842890625, + "7": 223.635484375, + "8": 242.8098359375, + "9": 258.8877109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373443, + "answer_accuracy": 0.9853169326241135, + "avg_response_len": 190.3282094309355, + "resp_tokens_sum": 72135914, + "pass_at_k": { + "pass@1": 0.9853169326241135, + "pass@2": 0.9897258105369812, + "pass@4": 0.9923083119232688, + "pass@8": 0.9941980514053886, + "pass@16": 0.9957425992524692, + "pass@32": 0.9970000759387941, + "pass@64": 0.9979621067463534, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391823377581121, + "pass@2": 0.9555910737927675, + "pass@4": 0.9660047825880169, + "pass@8": 0.9738180041000608, + "pass@16": 0.9800656176304551, + "pass@32": 0.9851412943559911, + "pass@64": 0.9900499064650484, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9965835244648318, + "pass@2": 0.9979603861157262, + "pass@4": 0.9991562422243091, + "pass@8": 0.9998166051472052, + "pass@16": 0.9999914663753303, + "pass@32": 0.9999999923331603, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9919677734375, + "pass@2": 0.994505874753937, + "pass@4": 0.9960995925314023, + "pass@8": 0.996883546028075, + "pass@16": 0.9972569049900324, + "pass@32": 0.9976562421654481, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9718998015873016, + "pass@2": 0.9793103987001622, + "pass@4": 0.9850510218960725, + "pass@8": 0.9893542220713245, + "pass@16": 0.9928501930138462, + "pass@32": 0.9955719071948553, + "pass@64": 0.9967508135601297, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9987369011976048, + "pass@2": 0.999835712433401, + "pass@4": 0.9999955036496185, + "pass@8": 0.9999999967220708, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9951923076923077, + "pass@2": 0.9970130282346363, + "pass@4": 0.9973102102067124, + "pass@8": 0.9975660630250558, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9898587740384616, + "pass@2": 0.9922923922370281, + "pass@4": 0.99351267930451, + "pass@8": 0.9945172949851099, + "pass@16": 0.9953777391919053, + "pass@32": 0.9962470240143486, + "pass@64": 0.9967505650677344, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9856073943661972, + "pass@2": 0.9916740601086839, + "pass@4": 0.9941322602280348, + "pass@8": 0.9961691775038143, + "pass@16": 0.9982934039427946, + "pass@32": 0.9996863387483557, + "pass@64": 0.9999950548051315, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 384404, + "answer_accuracy": 0.9839961500655308, + "avg_response_len": 197.60710446019004, + "resp_tokens_sum": 77196401, + "pass_at_k": { + "pass@1": 0.9839961500655308, + "pass@2": 0.9876649637903637, + "pass@4": 0.9901151083076449, + "pass@8": 0.9922382966768272, + "pass@16": 0.9940796597915759, + "pass@32": 0.9955590038375404, + "pass@64": 0.9967455136261166, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9214675996932515, + "pass@2": 0.9391171471184967, + "pass@4": 0.9516764283874023, + "pass@8": 0.9624968327655176, + "pass@16": 0.9718618878577276, + "pass@32": 0.9792168300320596, + "pass@64": 0.9850801174678696, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998626893939394, + "pass@2": 0.9996435814841327, + "pass@4": 0.9999705576575655, + "pass@8": 0.9999997974307437, + "pass@16": 0.9999999999966537, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981740014792899, + "pass@2": 0.9989069124772865, + "pass@4": 0.9996044064203513, + "pass@8": 0.9999514797911091, + "pass@16": 0.9999994548749709, + "pass@32": 0.9999999999843422, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9880434782608696, + "pass@2": 0.9894759928106812, + "pass@4": 0.9906430500535259, + "pass@8": 0.9922492237867451, + "pass@16": 0.9942422853173886, + "pass@32": 0.9960683553524285, + "pass@64": 0.9978312116817541, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9957691618911175, + "pass@2": 0.9972161039415202, + "pass@4": 0.9985833070114087, + "pass@8": 0.9995848817883625, + "pass@16": 0.9999582667747215, + "pass@32": 0.9999997227031818, + "pass@64": 0.9999999999997635, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934067919075145, + "pass@2": 0.9943153048336444, + "pass@4": 0.9945662773005977, + "pass@8": 0.9948843355239848, + "pass@16": 0.9954271919044801, + "pass@32": 0.9962098415443734, + "pass@64": 0.9969376905921441, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9842869718309859, + "pass@2": 0.9885327714317401, + "pass@4": 0.9907316494769141, + "pass@8": 0.9924513997074341, + "pass@16": 0.994185499597681, + "pass@32": 0.9957356444400437, + "pass@64": 0.9968305816745483, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9739193488023952, + "pass@2": 0.9800254314206235, + "pass@4": 0.9838955915503079, + "pass@8": 0.9874923584328539, + "pass@16": 0.990274389822248, + "pass@32": 0.9922249950791835, + "pass@64": 0.9936051611464493, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 376935, + "answer_accuracy": 0.9858736817877469, + "avg_response_len": 210.87395118429862, + "resp_tokens_sum": 80624703, + "pass_at_k": { + "pass@1": 0.9858736817877469, + "pass@2": 0.98984823342094, + "pass@4": 0.9923882188395682, + "pass@8": 0.9942328369269289, + "pass@16": 0.9956216488563557, + "pass@32": 0.9966827967430324, + "pass@64": 0.9973362892130396, + "pass@128": 0.9976565115500502 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392490671641791, + "pass@2": 0.9576459484075689, + "pass@4": 0.9704141928154498, + "pass@8": 0.9793468181522132, + "pass@16": 0.985302985372977, + "pass@32": 0.9894012615971982, + "pass@64": 0.9921077148584895, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995444606413995, + "pass@2": 0.9999386635981726, + "pass@4": 0.9999989407300763, + "pass@8": 0.9999999998458732, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9779411764705882, + "pass@2": 0.981991083835109, + "pass@4": 0.9851337267584199, + "pass@8": 0.9878139985800561, + "pass@16": 0.9901047493449984, + "pass@32": 0.9919331154908292, + "pass@64": 0.9933474491433287, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9968454258675079, + "pass@2": 0.9970879958518594, + "pass@4": 0.9973154095091425, + "pass@8": 0.9977276309284479, + "pass@16": 0.9984004097661192, + "pass@32": 0.9992711833733717, + "pass@64": 0.9999090886509849, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9894926819620253, + "pass@2": 0.9918008852038273, + "pass@4": 0.9932605224188747, + "pass@8": 0.9945238804517248, + "pass@16": 0.9955957102132315, + "pass@32": 0.9964363145736663, + "pass@64": 0.9968146458151399, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9874483858858859, + "pass@2": 0.9915703104679483, + "pass@4": 0.993921226625951, + "pass@8": 0.9955619363814269, + "pass@16": 0.9965104736117641, + "pass@32": 0.9969397427528214, + "pass@64": 0.9969968106564769, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9882008443679269, + "pass@4": 0.991108273141902, + "pass@8": 0.9928109656376645, + "pass@16": 0.9944391904203503, + "pass@32": 0.9960162607658117, + "pass@64": 0.9967354712683942, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4085625, + "avg_response_len": 322.294609375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4085625, + "pass@2": 0.47077454478346636, + "pass@4": 0.5240769172478387, + "pass@8": 0.5699273437755754, + "pass@16": 0.6092217796858977, + "pass@32": 0.6431335553597222, + "pass@64": 0.6733782070994079, + "pass@128": 0.7012 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.895703125, + "pass@2": 0.9253554379921274, + "pass@4": 0.9434871788526439, + "pass@8": 0.9559754198680026, + "pass@16": 0.9651618791630681, + "pass@32": 0.9719136760967422, + "pass@64": 0.9764095964514855, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.817328125, + "pass@2": 0.8694066190944905, + "pass@4": 0.8993614866891636, + "pass@8": 0.9194259948968868, + "pass@16": 0.9336959432236168, + "pass@32": 0.944403910581117, + "pass@64": 0.9539326803651079, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.61034375, + "pass@2": 0.6923665108267727, + "pass@4": 0.7510189754405697, + "pass@8": 0.7965151639164585, + "pass@16": 0.8348619428526896, + "pass@32": 0.8670921656599172, + "pass@64": 0.8938207509233816, + "pass@128": 0.918 + }, + "14": { + "pass@1": 0.4651640625, + "pass@2": 0.5500311269685039, + "pass@4": 0.6221303690476195, + "pass@8": 0.6816968014743139, + "pass@16": 0.7292125547128607, + "pass@32": 0.7677377492252492, + "pass@64": 0.801596238584059, + "pass@128": 0.831 + }, + "15": { + "pass@1": 0.3559765625, + "pass@2": 0.4363366141732286, + "pass@4": 0.5069854565054364, + "pass@8": 0.5686056990724146, + "pass@16": 0.6204541473118429, + "pass@32": 0.6627642449090546, + "pass@64": 0.699174624932904, + "pass@128": 0.734 + }, + "16": { + "pass@1": 0.2440234375, + "pass@2": 0.31041769192913415, + "pass@4": 0.3736180812710908, + "pass@8": 0.4314079756634686, + "pass@16": 0.4826026733494971, + "pass@32": 0.5285489205797774, + "pass@64": 0.570969046715714, + "pass@128": 0.611 + }, + "17": { + "pass@1": 0.207640625, + "pass@2": 0.2675178395669291, + "pass@4": 0.3226893885451819, + "pass@8": 0.37229302304476997, + "pass@16": 0.41801536555978624, + "pass@32": 0.46129380884292603, + "pass@64": 0.5021723071375973, + "pass@128": 0.541 + }, + "18": { + "pass@1": 0.185921875, + "pass@2": 0.2462491387795275, + "pass@4": 0.3059752948068988, + "pass@8": 0.3601648302492153, + "pass@16": 0.4074904294278328, + "pass@32": 0.4502261348579164, + "pass@64": 0.4901920079926209, + "pass@128": 0.527 + }, + "19": { + "pass@1": 0.15334375, + "pass@2": 0.20373535925196853, + "pass@4": 0.25254269291338566, + "pass@8": 0.29702385654594304, + "pass@16": 0.33673769557361394, + "pass@32": 0.3726330256503044, + "pass@64": 0.406919093736239, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.1501796875, + "pass@2": 0.20632910925196843, + "pass@4": 0.262960248406449, + "pass@8": 0.3161646730242484, + "pass@16": 0.36398516568404216, + "pass@32": 0.40472191719421413, + "pass@64": 0.4385957241549278, + "pass@128": 0.468 + } + }, + "per_op_accuracy": { + "11": 0.895703125, + "12": 0.817328125, + "13": 0.61034375, + "14": 0.4651640625, + "15": 0.3559765625, + "16": 0.2440234375, + "17": 0.207640625, + "18": 0.185921875, + "19": 0.15334375, + "20": 0.1501796875 + }, + "per_op_avg_response_len": { + "11": 296.1864375, + "12": 304.8280625, + "13": 316.6989609375, + "14": 327.34746875, + "15": 331.5959765625, + "16": 323.4783359375, + "17": 331.4318984375, + "18": 334.77853125, + "19": 325.3072890625, + "20": 331.2931328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.1864375, + "12": 304.8280625, + "13": 316.6989609375, + "14": 327.34746875, + "15": 331.5959765625, + "16": 323.4783359375, + "17": 331.4318984375, + "18": 334.77853125, + "19": 325.3072890625, + "20": 331.2931328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 176375, + "answer_accuracy": 0.4094887630014859, + "avg_response_len": 306.1256198922734, + "resp_tokens_sum": 131854427, + "pass_at_k": { + "pass@1": 0.4094887630014859, + "pass@2": 0.4709973997028252, + "pass@4": 0.5229526304104276, + "pass@8": 0.5668042637178171, + "pass@16": 0.6041985694987921, + "pass@32": 0.636782536620373, + "pass@64": 0.6667020846395655, + "pass@128": 0.6942050520059435 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8845609065155807, + "pass@2": 0.9171691324083782, + "pass@4": 0.937542806529495, + "pass@8": 0.9508063074854183, + "pass@16": 0.9599719123129501, + "pass@32": 0.966641033390473, + "pass@64": 0.9719494093940462, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.8151262358757062, + "pass@2": 0.8710546510076065, + "pass@4": 0.9037395913434542, + "pass@8": 0.9245161805980971, + "pass@16": 0.9387979429132806, + "pass@32": 0.9502059485422342, + "pass@64": 0.9611269600828399, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.6503967476489029, + "pass@2": 0.7226769802285681, + "pass@4": 0.7708130849827153, + "pass@8": 0.8068838086458526, + "pass@16": 0.8377186023812486, + "pass@32": 0.8647828944901396, + "pass@64": 0.8868747495654061, + "pass@128": 0.9028213166144201 + }, + "14": { + "pass@1": 0.4492309190031153, + "pass@2": 0.536244848774744, + "pass@4": 0.6107533541368071, + "pass@8": 0.6729903804799278, + "pass@16": 0.7238898678455619, + "pass@32": 0.7657076530948376, + "pass@64": 0.8026484654353564, + "pass@128": 0.8348909657320872 + }, + "15": { + "pass@1": 0.33808255057803466, + "pass@2": 0.42231770003641184, + "pass@4": 0.4959760785862749, + "pass@8": 0.5578812140172338, + "pass@16": 0.6085112269588493, + "pass@32": 0.649863667626754, + "pass@64": 0.6885207044236576, + "pass@128": 0.7283236994219653 + }, + "16": { + "pass@1": 0.2580141129032258, + "pass@2": 0.32291481457962906, + "pass@4": 0.38189584799883863, + "pass@8": 0.43668250574592865, + "pass@16": 0.4877986196385206, + "pass@32": 0.5350754320892568, + "pass@64": 0.5796115271331238, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19903811416184972, + "pass@2": 0.2558066599153426, + "pass@4": 0.304376596530925, + "pass@8": 0.34527377796485814, + "pass@16": 0.38477321578452844, + "pass@32": 0.4236055963938994, + "pass@64": 0.4586427605151552, + "pass@128": 0.4913294797687861 + }, + "18": { + "pass@1": 0.17020089285714285, + "pass@2": 0.22687113329583794, + "pass@4": 0.284835184530505, + "pass@8": 0.33964951048992603, + "pass@16": 0.38749462201555357, + "pass@32": 0.42923746107416955, + "pass@64": 0.4691776498182551, + "pass@128": 0.5085714285714286 + }, + "19": { + "pass@1": 0.1510885885885886, + "pass@2": 0.20444140006147882, + "pass@4": 0.2563637290271149, + "pass@8": 0.3013148135097076, + "pass@16": 0.33844867820314495, + "pass@32": 0.3717860620109835, + "pass@64": 0.4058951362319883, + "pass@128": 0.44144144144144143 + }, + "20": { + "pass@1": 0.14935247747747749, + "pass@2": 0.20391934749710333, + "pass@4": 0.2599002993432126, + "pass@8": 0.31194828909021904, + "pass@16": 0.35766435724738577, + "pass@32": 0.39695653407370624, + "pass@64": 0.4311676781647714, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 173003, + "answer_accuracy": 0.4018988812072554, + "avg_response_len": 317.94530785385075, + "resp_tokens_sum": 136864009, + "pass_at_k": { + "pass@1": 0.4018988812072554, + "pass@2": 0.463989343270094, + "pass@4": 0.5180412478825978, + "pass@8": 0.5653273197213595, + "pass@16": 0.6061284221321459, + "pass@32": 0.6410975186148631, + "pass@64": 0.6714973149431653, + "pass@128": 0.7008623253047874 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9010730421686747, + "pass@2": 0.9309511668722137, + "pass@4": 0.94928796739263, + "pass@8": 0.9615702566179654, + "pass@16": 0.9704520093607887, + "pass@32": 0.9770101498891585, + "pass@64": 0.9811200754082968, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8153653996865203, + "pass@2": 0.8600653648211684, + "pass@4": 0.8839596857798726, + "pass@8": 0.8995044053649817, + "pass@16": 0.9113281382028762, + "pass@32": 0.9220537842311385, + "pass@64": 0.932858148550826, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.5956101190476191, + "pass@2": 0.6822665089657906, + "pass@4": 0.7436430270270832, + "pass@8": 0.7923138314257233, + "pass@16": 0.834020503058109, + "pass@32": 0.8680755333396085, + "pass@64": 0.8957560237835692, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.44876187865497075, + "pass@2": 0.5299387432656442, + "pass@4": 0.6030156521005047, + "pass@8": 0.6656934204229993, + "pass@16": 0.7154344608861362, + "pass@32": 0.7549094623231906, + "pass@64": 0.7899431718135516, + "pass@128": 0.8245614035087719 + }, + "15": { + "pass@1": 0.3502604166666667, + "pass@2": 0.4297866708422811, + "pass@4": 0.4989867383054386, + "pass@8": 0.5591623054067931, + "pass@16": 0.610669142731308, + "pass@32": 0.6535868214261762, + "pass@64": 0.6893952778031021, + "pass@128": 0.7242424242424242 + }, + "16": { + "pass@1": 0.2311030982905983, + "pass@2": 0.2961883370347935, + "pass@4": 0.36089806936526087, + "pass@8": 0.42141125196089657, + "pass@16": 0.47406148705221673, + "pass@32": 0.5192663336859177, + "pass@64": 0.5586268992256944, + "pass@128": 0.5982905982905983 + }, + "17": { + "pass@1": 0.2037315605590062, + "pass@2": 0.2666669468626204, + "pass@4": 0.3291396813946393, + "pass@8": 0.3877869814725279, + "pass@16": 0.43914297265071023, + "pass@32": 0.4837502422979969, + "pass@64": 0.5249537984210485, + "pass@128": 0.5683229813664596 + }, + "18": { + "pass@1": 0.19767922794117648, + "pass@2": 0.26071857630847617, + "pass@4": 0.32037636223780847, + "pass@8": 0.37259126162145473, + "pass@16": 0.41917301613777475, + "pass@32": 0.46305120882097195, + "pass@64": 0.5028386527747987, + "pass@128": 0.538235294117647 + }, + "19": { + "pass@1": 0.13690476190476192, + "pass@2": 0.18360876277184102, + "pass@4": 0.23021844256075147, + "pass@8": 0.27379176875424555, + "pass@16": 0.3154033545548662, + "pass@32": 0.3563667081928054, + "pass@64": 0.3958945703686735, + "pass@128": 0.43154761904761907 + }, + "20": { + "pass@1": 0.15180576347305388, + "pass@2": 0.21004791597906552, + "pass@4": 0.26858944540615054, + "pass@8": 0.3248741814410061, + "pass@16": 0.3750522345917365, + "pass@32": 0.4147787110353963, + "pass@64": 0.4442726541841486, + "pass@128": 0.46706586826347307 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 173582, + "answer_accuracy": 0.4144588554400978, + "avg_response_len": 343.3934329156479, + "resp_tokens_sum": 143818664, + "pass_at_k": { + "pass@1": 0.4144588554400978, + "pass@2": 0.4775192654087188, + "pass@4": 0.5314366914786643, + "pass@8": 0.5778671497928926, + "pass@16": 0.6175671536873496, + "pass@32": 0.651757751457169, + "pass@64": 0.682177287905872, + "pass@128": 0.7087408312958435 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9025297619047619, + "pass@2": 0.9286315773028374, + "pass@4": 0.9440348030900895, + "pass@8": 0.9558713273920175, + "pass@16": 0.9654023079644926, + "pass@32": 0.9724508620530294, + "pass@64": 0.9764431425391512, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8216265290519877, + "pass@2": 0.8767352332105277, + "pass@4": 0.9096468855957216, + "pass@8": 0.9333497298279265, + "pass@16": 0.94999325793761, + "pass@32": 0.9599261395334334, + "pass@64": 0.9667033244895071, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5871431327160493, + "pass@2": 0.673652501336638, + "pass@4": 0.7396575329704156, + "pass@8": 0.7909357751217551, + "pass@16": 0.8329765095720012, + "pass@32": 0.868282274430008, + "pass@64": 0.8985271769175391, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.4969862759643917, + "pass@2": 0.5835533540503287, + "pass@4": 0.6523655470365901, + "pass@8": 0.7062306811738609, + "pass@16": 0.7482650489951872, + "pass@32": 0.7826900785379094, + "pass@64": 0.8124199300269259, + "pass@128": 0.8338278931750742 + }, + "15": { + "pass@1": 0.38090760030864196, + "pass@2": 0.45797866846505275, + "pass@4": 0.5268892273882428, + "pass@8": 0.5896766612290436, + "pass@16": 0.6431742150702188, + "pass@32": 0.6858881630850631, + "pass@64": 0.7205124059484368, + "pass@128": 0.75 + }, + "16": { + "pass@1": 0.24460730088495575, + "pass@2": 0.3137226935405197, + "pass@4": 0.3792187198945266, + "pass@8": 0.4369352490972164, + "pass@16": 0.48669474721601347, + "pass@32": 0.5321918982547222, + "pass@64": 0.5758449311983089, + "pass@128": 0.616519174041298 + }, + "17": { + "pass@1": 0.2203972138554217, + "pass@2": 0.2805481275495683, + "pass@4": 0.33551839980845743, + "pass@8": 0.3854243609481182, + "pass@16": 0.4321680593518407, + "pass@32": 0.4787912484056684, + "pass@64": 0.5254421352643848, + "pass@128": 0.5662650602409639 + }, + "18": { + "pass@1": 0.19077620967741934, + "pass@2": 0.2522578263906527, + "pass@4": 0.31404844213021765, + "pass@8": 0.36969829879498994, + "pass@16": 0.41725318140498613, + "pass@32": 0.45985681446073245, + "pass@64": 0.5000473826219357, + "pass@128": 0.535483870967742 + }, + "19": { + "pass@1": 0.1722998489425982, + "pass@2": 0.22345567595213742, + "pass@4": 0.271360043646357, + "pass@8": 0.3162899980235169, + "pass@16": 0.35667298671157666, + "pass@32": 0.38999713911741446, + "pass@64": 0.41914029524807356, + "pass@128": 0.44410876132930516 + }, + "20": { + "pass@1": 0.14937593843843844, + "pass@2": 0.20500889669196756, + "pass@4": 0.26037409597448957, + "pass@8": 0.3116453938615909, + "pass@16": 0.3592056707718412, + "pass@32": 0.40240030589143394, + "pass@64": 0.4403297922779381, + "pass@128": 0.4774774774774775 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6816373355263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6816373355263158, + "pass@2": 0.7162817745026797, + "pass@4": 0.7455310002269252, + "pass@8": 0.770588349122406, + "pass@16": 0.7920244536875509, + "pass@32": 0.8104731774755697, + "pass@64": 0.8268346669755965, + "pass@128": 0.8417368421052631 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9334296875, + "pass@2": 0.9509089566929134, + "pass@4": 0.962810891544807, + "pass@8": 0.971979454952471, + "pass@16": 0.9791457199182901, + "pass@32": 0.984637008012194, + "pass@64": 0.9891191210637711, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982734375, + "pass@2": 0.9991943897637795, + "pass@4": 0.9997140119047618, + "pass@8": 0.999939962982416, + "pass@16": 0.9999972095036287, + "pass@32": 0.9999999974929434, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996796875, + "pass@2": 0.9978724163385826, + "pass@4": 0.9986181589801275, + "pass@8": 0.998986334898379, + "pass@16": 0.9991220253445506, + "pass@32": 0.9992499974876511, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9795234375, + "pass@2": 0.9837289616141732, + "pass@4": 0.987008391263592, + "pass@8": 0.9898293216761134, + "pass@16": 0.99239701401116, + "pass@32": 0.9945059926298492, + "pass@64": 0.995966407010378, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9971015625, + "pass@2": 0.9980504429133858, + "pass@4": 0.9986530571803525, + "pass@8": 0.9991347816536281, + "pass@16": 0.9994783650002377, + "pass@32": 0.9997688683527692, + "pass@64": 0.9999711811022796, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9927734375, + "pass@2": 0.9944325787401576, + "pass@4": 0.99508110808024, + "pass@8": 0.9956768556165126, + "pass@16": 0.996358401907696, + "pass@32": 0.9971439391473108, + "pass@64": 0.9978118217783716, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.987078125, + "pass@2": 0.9907172736220474, + "pass@4": 0.9926614599737532, + "pass@8": 0.9941317677465087, + "pass@16": 0.9953316946977685, + "pass@32": 0.9962961596053821, + "pass@64": 0.9968609707442045, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9805078125, + "pass@2": 0.9867032480314963, + "pass@4": 0.9897727529058868, + "pass@8": 0.9922267160437407, + "pass@16": 0.9944163928210517, + "pass@32": 0.996052855710281, + "pass@64": 0.9968470998432064, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.895703125, + "pass@2": 0.9253554379921274, + "pass@4": 0.9434871788526439, + "pass@8": 0.9559754198680026, + "pass@16": 0.9651618791630681, + "pass@32": 0.9719136760967422, + "pass@64": 0.9764095964514855, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.817328125, + "pass@2": 0.8694066190944905, + "pass@4": 0.8993614866891636, + "pass@8": 0.9194259948968868, + "pass@16": 0.9336959432236168, + "pass@32": 0.944403910581117, + "pass@64": 0.9539326803651079, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.61034375, + "pass@2": 0.6923665108267727, + "pass@4": 0.7510189754405697, + "pass@8": 0.7965151639164585, + "pass@16": 0.8348619428526896, + "pass@32": 0.8670921656599172, + "pass@64": 0.8938207509233816, + "pass@128": 0.918 + }, + "14": { + "pass@1": 0.4651640625, + "pass@2": 0.5500311269685039, + "pass@4": 0.6221303690476195, + "pass@8": 0.6816968014743139, + "pass@16": 0.7292125547128607, + "pass@32": 0.7677377492252492, + "pass@64": 0.801596238584059, + "pass@128": 0.831 + }, + "15": { + "pass@1": 0.3559765625, + "pass@2": 0.4363366141732286, + "pass@4": 0.5069854565054364, + "pass@8": 0.5686056990724146, + "pass@16": 0.6204541473118429, + "pass@32": 0.6627642449090546, + "pass@64": 0.699174624932904, + "pass@128": 0.734 + }, + "16": { + "pass@1": 0.2440234375, + "pass@2": 0.31041769192913415, + "pass@4": 0.3736180812710908, + "pass@8": 0.4314079756634686, + "pass@16": 0.4826026733494971, + "pass@32": 0.5285489205797774, + "pass@64": 0.570969046715714, + "pass@128": 0.611 + }, + "17": { + "pass@1": 0.207640625, + "pass@2": 0.2675178395669291, + "pass@4": 0.3226893885451819, + "pass@8": 0.37229302304476997, + "pass@16": 0.41801536555978624, + "pass@32": 0.46129380884292603, + "pass@64": 0.5021723071375973, + "pass@128": 0.541 + }, + "18": { + "pass@1": 0.185921875, + "pass@2": 0.2462491387795275, + "pass@4": 0.3059752948068988, + "pass@8": 0.3601648302492153, + "pass@16": 0.4074904294278328, + "pass@32": 0.4502261348579164, + "pass@64": 0.4901920079926209, + "pass@128": 0.527 + }, + "19": { + "pass@1": 0.15334375, + "pass@2": 0.20373535925196853, + "pass@4": 0.25254269291338566, + "pass@8": 0.29702385654594304, + "pass@16": 0.33673769557361394, + "pass@32": 0.3726330256503044, + "pass@64": 0.406919093736239, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.1501796875, + "pass@2": 0.20632910925196843, + "pass@4": 0.262960248406449, + "pass@8": 0.3161646730242484, + "pass@16": 0.36398516568404216, + "pass@32": 0.40472191719421413, + "pass@64": 0.4385957241549278, + "pass@128": 0.468 + } + }, + "per_op_accuracy": { + "10": 0.9334296875, + "2": 1.0, + "3": 0.9982734375, + "4": 0.996796875, + "5": 0.9795234375, + "6": 0.9971015625, + "7": 0.9927734375, + "8": 0.987078125, + "9": 0.9805078125, + "11": 0.895703125, + "12": 0.817328125, + "13": 0.61034375, + "14": 0.4651640625, + "15": 0.3559765625, + "16": 0.2440234375, + "17": 0.207640625, + "18": 0.185921875, + "19": 0.15334375, + "20": 0.1501796875 + }, + "per_op_avg_response_len": { + "10": 279.63134375, + "2": 115.39703125, + "3": 144.4703671875, + "4": 159.7310078125, + "5": 173.3921328125, + "6": 198.5842890625, + "7": 223.635484375, + "8": 242.8098359375, + "9": 258.8877109375, + "11": 296.1864375, + "12": 304.8280625, + "13": 316.6989609375, + "14": 327.34746875, + "15": 331.5959765625, + "16": 323.4783359375, + "17": 331.4318984375, + "18": 334.77853125, + "19": 325.3072890625, + "20": 331.2931328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.63134375, + "2": 115.39703125, + "3": 144.4703671875, + "4": 159.7310078125, + "5": 173.3921328125, + "6": 198.5842890625, + "7": 223.635484375, + "8": 242.8098359375, + "9": 258.8877109375, + "11": 296.1864375, + "12": 304.8280625, + "13": 316.6989609375, + "14": 327.34746875, + "15": 331.5959765625, + "16": 323.4783359375, + "17": 331.4318984375, + "18": 334.77853125, + "19": 325.3072890625, + "20": 331.2931328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 549818, + "answer_accuracy": 0.679015669459374, + "avg_response_len": 251.92452403770156, + "resp_tokens_sum": 203990341, + "pass_at_k": { + "pass@1": 0.679015669459374, + "pass@2": 0.7137977197281024, + "pass@4": 0.7426431414694715, + "pass@8": 0.766853742905756, + "pass@16": 0.7874679138081008, + "pass@32": 0.8053889441325215, + "pass@64": 0.8217543965994494, + "pass@128": 0.8367056591843187 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391823377581121, + "pass@2": 0.9555910737927675, + "pass@4": 0.9660047825880169, + "pass@8": 0.9738180041000608, + "pass@16": 0.9800656176304551, + "pass@32": 0.9851412943559911, + "pass@64": 0.9900499064650484, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9965835244648318, + "pass@2": 0.9979603861157262, + "pass@4": 0.9991562422243091, + "pass@8": 0.9998166051472052, + "pass@16": 0.9999914663753303, + "pass@32": 0.9999999923331603, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9919677734375, + "pass@2": 0.994505874753937, + "pass@4": 0.9960995925314023, + "pass@8": 0.996883546028075, + "pass@16": 0.9972569049900324, + "pass@32": 0.9976562421654481, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9718998015873016, + "pass@2": 0.9793103987001622, + "pass@4": 0.9850510218960725, + "pass@8": 0.9893542220713245, + "pass@16": 0.9928501930138462, + "pass@32": 0.9955719071948553, + "pass@64": 0.9967508135601297, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9987369011976048, + "pass@2": 0.999835712433401, + "pass@4": 0.9999955036496185, + "pass@8": 0.9999999967220708, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9951923076923077, + "pass@2": 0.9970130282346363, + "pass@4": 0.9973102102067124, + "pass@8": 0.9975660630250558, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9898587740384616, + "pass@2": 0.9922923922370281, + "pass@4": 0.99351267930451, + "pass@8": 0.9945172949851099, + "pass@16": 0.9953777391919053, + "pass@32": 0.9962470240143486, + "pass@64": 0.9967505650677344, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9856073943661972, + "pass@2": 0.9916740601086839, + "pass@4": 0.9941322602280348, + "pass@8": 0.9961691775038143, + "pass@16": 0.9982934039427946, + "pass@32": 0.9996863387483557, + "pass@64": 0.9999950548051315, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8845609065155807, + "pass@2": 0.9171691324083782, + "pass@4": 0.937542806529495, + "pass@8": 0.9508063074854183, + "pass@16": 0.9599719123129501, + "pass@32": 0.966641033390473, + "pass@64": 0.9719494093940462, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.8151262358757062, + "pass@2": 0.8710546510076065, + "pass@4": 0.9037395913434542, + "pass@8": 0.9245161805980971, + "pass@16": 0.9387979429132806, + "pass@32": 0.9502059485422342, + "pass@64": 0.9611269600828399, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.6503967476489029, + "pass@2": 0.7226769802285681, + "pass@4": 0.7708130849827153, + "pass@8": 0.8068838086458526, + "pass@16": 0.8377186023812486, + "pass@32": 0.8647828944901396, + "pass@64": 0.8868747495654061, + "pass@128": 0.9028213166144201 + }, + "14": { + "pass@1": 0.4492309190031153, + "pass@2": 0.536244848774744, + "pass@4": 0.6107533541368071, + "pass@8": 0.6729903804799278, + "pass@16": 0.7238898678455619, + "pass@32": 0.7657076530948376, + "pass@64": 0.8026484654353564, + "pass@128": 0.8348909657320872 + }, + "15": { + "pass@1": 0.33808255057803466, + "pass@2": 0.42231770003641184, + "pass@4": 0.4959760785862749, + "pass@8": 0.5578812140172338, + "pass@16": 0.6085112269588493, + "pass@32": 0.649863667626754, + "pass@64": 0.6885207044236576, + "pass@128": 0.7283236994219653 + }, + "16": { + "pass@1": 0.2580141129032258, + "pass@2": 0.32291481457962906, + "pass@4": 0.38189584799883863, + "pass@8": 0.43668250574592865, + "pass@16": 0.4877986196385206, + "pass@32": 0.5350754320892568, + "pass@64": 0.5796115271331238, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19903811416184972, + "pass@2": 0.2558066599153426, + "pass@4": 0.304376596530925, + "pass@8": 0.34527377796485814, + "pass@16": 0.38477321578452844, + "pass@32": 0.4236055963938994, + "pass@64": 0.4586427605151552, + "pass@128": 0.4913294797687861 + }, + "18": { + "pass@1": 0.17020089285714285, + "pass@2": 0.22687113329583794, + "pass@4": 0.284835184530505, + "pass@8": 0.33964951048992603, + "pass@16": 0.38749462201555357, + "pass@32": 0.42923746107416955, + "pass@64": 0.4691776498182551, + "pass@128": 0.5085714285714286 + }, + "19": { + "pass@1": 0.1510885885885886, + "pass@2": 0.20444140006147882, + "pass@4": 0.2563637290271149, + "pass@8": 0.3013148135097076, + "pass@16": 0.33844867820314495, + "pass@32": 0.3717860620109835, + "pass@64": 0.4058951362319883, + "pass@128": 0.44144144144144143 + }, + "20": { + "pass@1": 0.14935247747747749, + "pass@2": 0.20391934749710333, + "pass@4": 0.2599002993432126, + "pass@8": 0.31194828909021904, + "pass@16": 0.35766435724738577, + "pass@32": 0.39695653407370624, + "pass@64": 0.4311676781647714, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 557407, + "answer_accuracy": 0.678837441543258, + "avg_response_len": 260.69321171083396, + "resp_tokens_sum": 214060410, + "pass_at_k": { + "pass@1": 0.678837441543258, + "pass@2": 0.7131332238356208, + "pass@4": 0.7426350782827879, + "pass@8": 0.7684344594981463, + "pass@16": 0.7907000787707481, + "pass@32": 0.8097360927223639, + "pass@64": 0.8262373776680917, + "pass@128": 0.8419329696024942 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9214675996932515, + "pass@2": 0.9391171471184967, + "pass@4": 0.9516764283874023, + "pass@8": 0.9624968327655176, + "pass@16": 0.9718618878577276, + "pass@32": 0.9792168300320596, + "pass@64": 0.9850801174678696, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998626893939394, + "pass@2": 0.9996435814841327, + "pass@4": 0.9999705576575655, + "pass@8": 0.9999997974307437, + "pass@16": 0.9999999999966537, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981740014792899, + "pass@2": 0.9989069124772865, + "pass@4": 0.9996044064203513, + "pass@8": 0.9999514797911091, + "pass@16": 0.9999994548749709, + "pass@32": 0.9999999999843422, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9880434782608696, + "pass@2": 0.9894759928106812, + "pass@4": 0.9906430500535259, + "pass@8": 0.9922492237867451, + "pass@16": 0.9942422853173886, + "pass@32": 0.9960683553524285, + "pass@64": 0.9978312116817541, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9957691618911175, + "pass@2": 0.9972161039415202, + "pass@4": 0.9985833070114087, + "pass@8": 0.9995848817883625, + "pass@16": 0.9999582667747215, + "pass@32": 0.9999997227031818, + "pass@64": 0.9999999999997635, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934067919075145, + "pass@2": 0.9943153048336444, + "pass@4": 0.9945662773005977, + "pass@8": 0.9948843355239848, + "pass@16": 0.9954271919044801, + "pass@32": 0.9962098415443734, + "pass@64": 0.9969376905921441, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9842869718309859, + "pass@2": 0.9885327714317401, + "pass@4": 0.9907316494769141, + "pass@8": 0.9924513997074341, + "pass@16": 0.994185499597681, + "pass@32": 0.9957356444400437, + "pass@64": 0.9968305816745483, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9739193488023952, + "pass@2": 0.9800254314206235, + "pass@4": 0.9838955915503079, + "pass@8": 0.9874923584328539, + "pass@16": 0.990274389822248, + "pass@32": 0.9922249950791835, + "pass@64": 0.9936051611464493, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9010730421686747, + "pass@2": 0.9309511668722137, + "pass@4": 0.94928796739263, + "pass@8": 0.9615702566179654, + "pass@16": 0.9704520093607887, + "pass@32": 0.9770101498891585, + "pass@64": 0.9811200754082968, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8153653996865203, + "pass@2": 0.8600653648211684, + "pass@4": 0.8839596857798726, + "pass@8": 0.8995044053649817, + "pass@16": 0.9113281382028762, + "pass@32": 0.9220537842311385, + "pass@64": 0.932858148550826, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.5956101190476191, + "pass@2": 0.6822665089657906, + "pass@4": 0.7436430270270832, + "pass@8": 0.7923138314257233, + "pass@16": 0.834020503058109, + "pass@32": 0.8680755333396085, + "pass@64": 0.8957560237835692, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.44876187865497075, + "pass@2": 0.5299387432656442, + "pass@4": 0.6030156521005047, + "pass@8": 0.6656934204229993, + "pass@16": 0.7154344608861362, + "pass@32": 0.7549094623231906, + "pass@64": 0.7899431718135516, + "pass@128": 0.8245614035087719 + }, + "15": { + "pass@1": 0.3502604166666667, + "pass@2": 0.4297866708422811, + "pass@4": 0.4989867383054386, + "pass@8": 0.5591623054067931, + "pass@16": 0.610669142731308, + "pass@32": 0.6535868214261762, + "pass@64": 0.6893952778031021, + "pass@128": 0.7242424242424242 + }, + "16": { + "pass@1": 0.2311030982905983, + "pass@2": 0.2961883370347935, + "pass@4": 0.36089806936526087, + "pass@8": 0.42141125196089657, + "pass@16": 0.47406148705221673, + "pass@32": 0.5192663336859177, + "pass@64": 0.5586268992256944, + "pass@128": 0.5982905982905983 + }, + "17": { + "pass@1": 0.2037315605590062, + "pass@2": 0.2666669468626204, + "pass@4": 0.3291396813946393, + "pass@8": 0.3877869814725279, + "pass@16": 0.43914297265071023, + "pass@32": 0.4837502422979969, + "pass@64": 0.5249537984210485, + "pass@128": 0.5683229813664596 + }, + "18": { + "pass@1": 0.19767922794117648, + "pass@2": 0.26071857630847617, + "pass@4": 0.32037636223780847, + "pass@8": 0.37259126162145473, + "pass@16": 0.41917301613777475, + "pass@32": 0.46305120882097195, + "pass@64": 0.5028386527747987, + "pass@128": 0.538235294117647 + }, + "19": { + "pass@1": 0.13690476190476192, + "pass@2": 0.18360876277184102, + "pass@4": 0.23021844256075147, + "pass@8": 0.27379176875424555, + "pass@16": 0.3154033545548662, + "pass@32": 0.3563667081928054, + "pass@64": 0.3958945703686735, + "pass@128": 0.43154761904761907 + }, + "20": { + "pass@1": 0.15180576347305388, + "pass@2": 0.21004791597906552, + "pass@4": 0.26858944540615054, + "pass@8": 0.3248741814410061, + "pass@16": 0.3750522345917365, + "pass@32": 0.4147787110353963, + "pass@64": 0.4442726541841486, + "pass@128": 0.46706586826347307 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 550517, + "answer_accuracy": 0.6871567442882249, + "avg_response_len": 280.15079161008146, + "resp_tokens_sum": 224443367, + "pass_at_k": { + "pass@1": 0.6871567442882249, + "pass@2": 0.7220194455417217, + "pass@4": 0.7514178725342621, + "pass@8": 0.7765705061548289, + "pass@16": 0.797987153219204, + "pass@32": 0.8163672913627276, + "pass@64": 0.8325814957513004, + "pass@128": 0.8466208659530277 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392490671641791, + "pass@2": 0.9576459484075689, + "pass@4": 0.9704141928154498, + "pass@8": 0.9793468181522132, + "pass@16": 0.985302985372977, + "pass@32": 0.9894012615971982, + "pass@64": 0.9921077148584895, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995444606413995, + "pass@2": 0.9999386635981726, + "pass@4": 0.9999989407300763, + "pass@8": 0.9999999998458732, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9779411764705882, + "pass@2": 0.981991083835109, + "pass@4": 0.9851337267584199, + "pass@8": 0.9878139985800561, + "pass@16": 0.9901047493449984, + "pass@32": 0.9919331154908292, + "pass@64": 0.9933474491433287, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9968454258675079, + "pass@2": 0.9970879958518594, + "pass@4": 0.9973154095091425, + "pass@8": 0.9977276309284479, + "pass@16": 0.9984004097661192, + "pass@32": 0.9992711833733717, + "pass@64": 0.9999090886509849, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9894926819620253, + "pass@2": 0.9918008852038273, + "pass@4": 0.9932605224188747, + "pass@8": 0.9945238804517248, + "pass@16": 0.9955957102132315, + "pass@32": 0.9964363145736663, + "pass@64": 0.9968146458151399, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9874483858858859, + "pass@2": 0.9915703104679483, + "pass@4": 0.993921226625951, + "pass@8": 0.9955619363814269, + "pass@16": 0.9965104736117641, + "pass@32": 0.9969397427528214, + "pass@64": 0.9969968106564769, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9882008443679269, + "pass@4": 0.991108273141902, + "pass@8": 0.9928109656376645, + "pass@16": 0.9944391904203503, + "pass@32": 0.9960162607658117, + "pass@64": 0.9967354712683942, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9025297619047619, + "pass@2": 0.9286315773028374, + "pass@4": 0.9440348030900895, + "pass@8": 0.9558713273920175, + "pass@16": 0.9654023079644926, + "pass@32": 0.9724508620530294, + "pass@64": 0.9764431425391512, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8216265290519877, + "pass@2": 0.8767352332105277, + "pass@4": 0.9096468855957216, + "pass@8": 0.9333497298279265, + "pass@16": 0.94999325793761, + "pass@32": 0.9599261395334334, + "pass@64": 0.9667033244895071, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5871431327160493, + "pass@2": 0.673652501336638, + "pass@4": 0.7396575329704156, + "pass@8": 0.7909357751217551, + "pass@16": 0.8329765095720012, + "pass@32": 0.868282274430008, + "pass@64": 0.8985271769175391, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.4969862759643917, + "pass@2": 0.5835533540503287, + "pass@4": 0.6523655470365901, + "pass@8": 0.7062306811738609, + "pass@16": 0.7482650489951872, + "pass@32": 0.7826900785379094, + "pass@64": 0.8124199300269259, + "pass@128": 0.8338278931750742 + }, + "15": { + "pass@1": 0.38090760030864196, + "pass@2": 0.45797866846505275, + "pass@4": 0.5268892273882428, + "pass@8": 0.5896766612290436, + "pass@16": 0.6431742150702188, + "pass@32": 0.6858881630850631, + "pass@64": 0.7205124059484368, + "pass@128": 0.75 + }, + "16": { + "pass@1": 0.24460730088495575, + "pass@2": 0.3137226935405197, + "pass@4": 0.3792187198945266, + "pass@8": 0.4369352490972164, + "pass@16": 0.48669474721601347, + "pass@32": 0.5321918982547222, + "pass@64": 0.5758449311983089, + "pass@128": 0.616519174041298 + }, + "17": { + "pass@1": 0.2203972138554217, + "pass@2": 0.2805481275495683, + "pass@4": 0.33551839980845743, + "pass@8": 0.3854243609481182, + "pass@16": 0.4321680593518407, + "pass@32": 0.4787912484056684, + "pass@64": 0.5254421352643848, + "pass@128": 0.5662650602409639 + }, + "18": { + "pass@1": 0.19077620967741934, + "pass@2": 0.2522578263906527, + "pass@4": 0.31404844213021765, + "pass@8": 0.36969829879498994, + "pass@16": 0.41725318140498613, + "pass@32": 0.45985681446073245, + "pass@64": 0.5000473826219357, + "pass@128": 0.535483870967742 + }, + "19": { + "pass@1": 0.1722998489425982, + "pass@2": 0.22345567595213742, + "pass@4": 0.271360043646357, + "pass@8": 0.3162899980235169, + "pass@16": 0.35667298671157666, + "pass@32": 0.38999713911741446, + "pass@64": 0.41914029524807356, + "pass@128": 0.44410876132930516 + }, + "20": { + "pass@1": 0.14937593843843844, + "pass@2": 0.20500889669196756, + "pass@4": 0.26037409597448957, + "pass@8": 0.3116453938615909, + "pass@16": 0.3592056707718412, + "pass@32": 0.40240030589143394, + "pass@64": 0.4403297922779381, + "pass@128": 0.4774774774774775 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7f788c21de5fae5f4e2f4b469110244d56976fa6 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43638251de277d07b3a1e61469c76008b512180d7f63f0e474015c692d10d84f +size 2556665284 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/summary.csv b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..923283d272ea4e58f3467939531352069ff66a22 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op7-10,0.9850538194444445,0.4085625,0.6816373355263158,0.0,0.0,0.0,199.6154670138889,322.294609375,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/summary.json b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7be5b880e7c91064a1905868175b30aafa75eb2e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op7-10", + "id_acc": 0.9850538194444445, + "ood_acc": 0.4085625, + "total_acc": 0.6816373355263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.6154670138889, + "ood_avg_resp_len": 322.294609375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 549818, + "answer_accuracy": 0.679015669459374, + "avg_response_len": 251.92452403770156, + "resp_tokens_sum": 203990341, + "pass_at_k": { + "pass@1": 0.679015669459374, + "pass@2": 0.7137977197281024, + "pass@4": 0.7426431414694715, + "pass@8": 0.766853742905756, + "pass@16": 0.7874679138081008, + "pass@32": 0.8053889441325215, + "pass@64": 0.8217543965994494, + "pass@128": 0.8367056591843187 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391823377581121, + "pass@2": 0.9555910737927675, + "pass@4": 0.9660047825880169, + "pass@8": 0.9738180041000608, + "pass@16": 0.9800656176304551, + "pass@32": 0.9851412943559911, + "pass@64": 0.9900499064650484, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9965835244648318, + "pass@2": 0.9979603861157262, + "pass@4": 0.9991562422243091, + "pass@8": 0.9998166051472052, + "pass@16": 0.9999914663753303, + "pass@32": 0.9999999923331603, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9919677734375, + "pass@2": 0.994505874753937, + "pass@4": 0.9960995925314023, + "pass@8": 0.996883546028075, + "pass@16": 0.9972569049900324, + "pass@32": 0.9976562421654481, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9718998015873016, + "pass@2": 0.9793103987001622, + "pass@4": 0.9850510218960725, + "pass@8": 0.9893542220713245, + "pass@16": 0.9928501930138462, + "pass@32": 0.9955719071948553, + "pass@64": 0.9967508135601297, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9987369011976048, + "pass@2": 0.999835712433401, + "pass@4": 0.9999955036496185, + "pass@8": 0.9999999967220708, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9951923076923077, + "pass@2": 0.9970130282346363, + "pass@4": 0.9973102102067124, + "pass@8": 0.9975660630250558, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9898587740384616, + "pass@2": 0.9922923922370281, + "pass@4": 0.99351267930451, + "pass@8": 0.9945172949851099, + "pass@16": 0.9953777391919053, + "pass@32": 0.9962470240143486, + "pass@64": 0.9967505650677344, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9856073943661972, + "pass@2": 0.9916740601086839, + "pass@4": 0.9941322602280348, + "pass@8": 0.9961691775038143, + "pass@16": 0.9982934039427946, + "pass@32": 0.9996863387483557, + "pass@64": 0.9999950548051315, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8845609065155807, + "pass@2": 0.9171691324083782, + "pass@4": 0.937542806529495, + "pass@8": 0.9508063074854183, + "pass@16": 0.9599719123129501, + "pass@32": 0.966641033390473, + "pass@64": 0.9719494093940462, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.8151262358757062, + "pass@2": 0.8710546510076065, + "pass@4": 0.9037395913434542, + "pass@8": 0.9245161805980971, + "pass@16": 0.9387979429132806, + "pass@32": 0.9502059485422342, + "pass@64": 0.9611269600828399, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.6503967476489029, + "pass@2": 0.7226769802285681, + "pass@4": 0.7708130849827153, + "pass@8": 0.8068838086458526, + "pass@16": 0.8377186023812486, + "pass@32": 0.8647828944901396, + "pass@64": 0.8868747495654061, + "pass@128": 0.9028213166144201 + }, + "14": { + "pass@1": 0.4492309190031153, + "pass@2": 0.536244848774744, + "pass@4": 0.6107533541368071, + "pass@8": 0.6729903804799278, + "pass@16": 0.7238898678455619, + "pass@32": 0.7657076530948376, + "pass@64": 0.8026484654353564, + "pass@128": 0.8348909657320872 + }, + "15": { + "pass@1": 0.33808255057803466, + "pass@2": 0.42231770003641184, + "pass@4": 0.4959760785862749, + "pass@8": 0.5578812140172338, + "pass@16": 0.6085112269588493, + "pass@32": 0.649863667626754, + "pass@64": 0.6885207044236576, + "pass@128": 0.7283236994219653 + }, + "16": { + "pass@1": 0.2580141129032258, + "pass@2": 0.32291481457962906, + "pass@4": 0.38189584799883863, + "pass@8": 0.43668250574592865, + "pass@16": 0.4877986196385206, + "pass@32": 0.5350754320892568, + "pass@64": 0.5796115271331238, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19903811416184972, + "pass@2": 0.2558066599153426, + "pass@4": 0.304376596530925, + "pass@8": 0.34527377796485814, + "pass@16": 0.38477321578452844, + "pass@32": 0.4236055963938994, + "pass@64": 0.4586427605151552, + "pass@128": 0.4913294797687861 + }, + "18": { + "pass@1": 0.17020089285714285, + "pass@2": 0.22687113329583794, + "pass@4": 0.284835184530505, + "pass@8": 0.33964951048992603, + "pass@16": 0.38749462201555357, + "pass@32": 0.42923746107416955, + "pass@64": 0.4691776498182551, + "pass@128": 0.5085714285714286 + }, + "19": { + "pass@1": 0.1510885885885886, + "pass@2": 0.20444140006147882, + "pass@4": 0.2563637290271149, + "pass@8": 0.3013148135097076, + "pass@16": 0.33844867820314495, + "pass@32": 0.3717860620109835, + "pass@64": 0.4058951362319883, + "pass@128": 0.44144144144144143 + }, + "20": { + "pass@1": 0.14935247747747749, + "pass@2": 0.20391934749710333, + "pass@4": 0.2599002993432126, + "pass@8": 0.31194828909021904, + "pass@16": 0.35766435724738577, + "pass@32": 0.39695653407370624, + "pass@64": 0.4311676781647714, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 557407, + "answer_accuracy": 0.678837441543258, + "avg_response_len": 260.69321171083396, + "resp_tokens_sum": 214060410, + "pass_at_k": { + "pass@1": 0.678837441543258, + "pass@2": 0.7131332238356208, + "pass@4": 0.7426350782827879, + "pass@8": 0.7684344594981463, + "pass@16": 0.7907000787707481, + "pass@32": 0.8097360927223639, + "pass@64": 0.8262373776680917, + "pass@128": 0.8419329696024942 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9214675996932515, + "pass@2": 0.9391171471184967, + "pass@4": 0.9516764283874023, + "pass@8": 0.9624968327655176, + "pass@16": 0.9718618878577276, + "pass@32": 0.9792168300320596, + "pass@64": 0.9850801174678696, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998626893939394, + "pass@2": 0.9996435814841327, + "pass@4": 0.9999705576575655, + "pass@8": 0.9999997974307437, + "pass@16": 0.9999999999966537, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981740014792899, + "pass@2": 0.9989069124772865, + "pass@4": 0.9996044064203513, + "pass@8": 0.9999514797911091, + "pass@16": 0.9999994548749709, + "pass@32": 0.9999999999843422, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9880434782608696, + "pass@2": 0.9894759928106812, + "pass@4": 0.9906430500535259, + "pass@8": 0.9922492237867451, + "pass@16": 0.9942422853173886, + "pass@32": 0.9960683553524285, + "pass@64": 0.9978312116817541, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9957691618911175, + "pass@2": 0.9972161039415202, + "pass@4": 0.9985833070114087, + "pass@8": 0.9995848817883625, + "pass@16": 0.9999582667747215, + "pass@32": 0.9999997227031818, + "pass@64": 0.9999999999997635, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934067919075145, + "pass@2": 0.9943153048336444, + "pass@4": 0.9945662773005977, + "pass@8": 0.9948843355239848, + "pass@16": 0.9954271919044801, + "pass@32": 0.9962098415443734, + "pass@64": 0.9969376905921441, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9842869718309859, + "pass@2": 0.9885327714317401, + "pass@4": 0.9907316494769141, + "pass@8": 0.9924513997074341, + "pass@16": 0.994185499597681, + "pass@32": 0.9957356444400437, + "pass@64": 0.9968305816745483, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9739193488023952, + "pass@2": 0.9800254314206235, + "pass@4": 0.9838955915503079, + "pass@8": 0.9874923584328539, + "pass@16": 0.990274389822248, + "pass@32": 0.9922249950791835, + "pass@64": 0.9936051611464493, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9010730421686747, + "pass@2": 0.9309511668722137, + "pass@4": 0.94928796739263, + "pass@8": 0.9615702566179654, + "pass@16": 0.9704520093607887, + "pass@32": 0.9770101498891585, + "pass@64": 0.9811200754082968, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8153653996865203, + "pass@2": 0.8600653648211684, + "pass@4": 0.8839596857798726, + "pass@8": 0.8995044053649817, + "pass@16": 0.9113281382028762, + "pass@32": 0.9220537842311385, + "pass@64": 0.932858148550826, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.5956101190476191, + "pass@2": 0.6822665089657906, + "pass@4": 0.7436430270270832, + "pass@8": 0.7923138314257233, + "pass@16": 0.834020503058109, + "pass@32": 0.8680755333396085, + "pass@64": 0.8957560237835692, + "pass@128": 0.9243697478991597 + }, + "14": { + "pass@1": 0.44876187865497075, + "pass@2": 0.5299387432656442, + "pass@4": 0.6030156521005047, + "pass@8": 0.6656934204229993, + "pass@16": 0.7154344608861362, + "pass@32": 0.7549094623231906, + "pass@64": 0.7899431718135516, + "pass@128": 0.8245614035087719 + }, + "15": { + "pass@1": 0.3502604166666667, + "pass@2": 0.4297866708422811, + "pass@4": 0.4989867383054386, + "pass@8": 0.5591623054067931, + "pass@16": 0.610669142731308, + "pass@32": 0.6535868214261762, + "pass@64": 0.6893952778031021, + "pass@128": 0.7242424242424242 + }, + "16": { + "pass@1": 0.2311030982905983, + "pass@2": 0.2961883370347935, + "pass@4": 0.36089806936526087, + "pass@8": 0.42141125196089657, + "pass@16": 0.47406148705221673, + "pass@32": 0.5192663336859177, + "pass@64": 0.5586268992256944, + "pass@128": 0.5982905982905983 + }, + "17": { + "pass@1": 0.2037315605590062, + "pass@2": 0.2666669468626204, + "pass@4": 0.3291396813946393, + "pass@8": 0.3877869814725279, + "pass@16": 0.43914297265071023, + "pass@32": 0.4837502422979969, + "pass@64": 0.5249537984210485, + "pass@128": 0.5683229813664596 + }, + "18": { + "pass@1": 0.19767922794117648, + "pass@2": 0.26071857630847617, + "pass@4": 0.32037636223780847, + "pass@8": 0.37259126162145473, + "pass@16": 0.41917301613777475, + "pass@32": 0.46305120882097195, + "pass@64": 0.5028386527747987, + "pass@128": 0.538235294117647 + }, + "19": { + "pass@1": 0.13690476190476192, + "pass@2": 0.18360876277184102, + "pass@4": 0.23021844256075147, + "pass@8": 0.27379176875424555, + "pass@16": 0.3154033545548662, + "pass@32": 0.3563667081928054, + "pass@64": 0.3958945703686735, + "pass@128": 0.43154761904761907 + }, + "20": { + "pass@1": 0.15180576347305388, + "pass@2": 0.21004791597906552, + "pass@4": 0.26858944540615054, + "pass@8": 0.3248741814410061, + "pass@16": 0.3750522345917365, + "pass@32": 0.4147787110353963, + "pass@64": 0.4442726541841486, + "pass@128": 0.46706586826347307 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 550517, + "answer_accuracy": 0.6871567442882249, + "avg_response_len": 280.15079161008146, + "resp_tokens_sum": 224443367, + "pass_at_k": { + "pass@1": 0.6871567442882249, + "pass@2": 0.7220194455417217, + "pass@4": 0.7514178725342621, + "pass@8": 0.7765705061548289, + "pass@16": 0.797987153219204, + "pass@32": 0.8163672913627276, + "pass@64": 0.8325814957513004, + "pass@128": 0.8466208659530277 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392490671641791, + "pass@2": 0.9576459484075689, + "pass@4": 0.9704141928154498, + "pass@8": 0.9793468181522132, + "pass@16": 0.985302985372977, + "pass@32": 0.9894012615971982, + "pass@64": 0.9921077148584895, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995444606413995, + "pass@2": 0.9999386635981726, + "pass@4": 0.9999989407300763, + "pass@8": 0.9999999998458732, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9779411764705882, + "pass@2": 0.981991083835109, + "pass@4": 0.9851337267584199, + "pass@8": 0.9878139985800561, + "pass@16": 0.9901047493449984, + "pass@32": 0.9919331154908292, + "pass@64": 0.9933474491433287, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9968454258675079, + "pass@2": 0.9970879958518594, + "pass@4": 0.9973154095091425, + "pass@8": 0.9977276309284479, + "pass@16": 0.9984004097661192, + "pass@32": 0.9992711833733717, + "pass@64": 0.9999090886509849, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9894926819620253, + "pass@2": 0.9918008852038273, + "pass@4": 0.9932605224188747, + "pass@8": 0.9945238804517248, + "pass@16": 0.9955957102132315, + "pass@32": 0.9964363145736663, + "pass@64": 0.9968146458151399, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9874483858858859, + "pass@2": 0.9915703104679483, + "pass@4": 0.993921226625951, + "pass@8": 0.9955619363814269, + "pass@16": 0.9965104736117641, + "pass@32": 0.9969397427528214, + "pass@64": 0.9969968106564769, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9882008443679269, + "pass@4": 0.991108273141902, + "pass@8": 0.9928109656376645, + "pass@16": 0.9944391904203503, + "pass@32": 0.9960162607658117, + "pass@64": 0.9967354712683942, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9025297619047619, + "pass@2": 0.9286315773028374, + "pass@4": 0.9440348030900895, + "pass@8": 0.9558713273920175, + "pass@16": 0.9654023079644926, + "pass@32": 0.9724508620530294, + "pass@64": 0.9764431425391512, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8216265290519877, + "pass@2": 0.8767352332105277, + "pass@4": 0.9096468855957216, + "pass@8": 0.9333497298279265, + "pass@16": 0.94999325793761, + "pass@32": 0.9599261395334334, + "pass@64": 0.9667033244895071, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5871431327160493, + "pass@2": 0.673652501336638, + "pass@4": 0.7396575329704156, + "pass@8": 0.7909357751217551, + "pass@16": 0.8329765095720012, + "pass@32": 0.868282274430008, + "pass@64": 0.8985271769175391, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.4969862759643917, + "pass@2": 0.5835533540503287, + "pass@4": 0.6523655470365901, + "pass@8": 0.7062306811738609, + "pass@16": 0.7482650489951872, + "pass@32": 0.7826900785379094, + "pass@64": 0.8124199300269259, + "pass@128": 0.8338278931750742 + }, + "15": { + "pass@1": 0.38090760030864196, + "pass@2": 0.45797866846505275, + "pass@4": 0.5268892273882428, + "pass@8": 0.5896766612290436, + "pass@16": 0.6431742150702188, + "pass@32": 0.6858881630850631, + "pass@64": 0.7205124059484368, + "pass@128": 0.75 + }, + "16": { + "pass@1": 0.24460730088495575, + "pass@2": 0.3137226935405197, + "pass@4": 0.3792187198945266, + "pass@8": 0.4369352490972164, + "pass@16": 0.48669474721601347, + "pass@32": 0.5321918982547222, + "pass@64": 0.5758449311983089, + "pass@128": 0.616519174041298 + }, + "17": { + "pass@1": 0.2203972138554217, + "pass@2": 0.2805481275495683, + "pass@4": 0.33551839980845743, + "pass@8": 0.3854243609481182, + "pass@16": 0.4321680593518407, + "pass@32": 0.4787912484056684, + "pass@64": 0.5254421352643848, + "pass@128": 0.5662650602409639 + }, + "18": { + "pass@1": 0.19077620967741934, + "pass@2": 0.2522578263906527, + "pass@4": 0.31404844213021765, + "pass@8": 0.36969829879498994, + "pass@16": 0.41725318140498613, + "pass@32": 0.45985681446073245, + "pass@64": 0.5000473826219357, + "pass@128": 0.535483870967742 + }, + "19": { + "pass@1": 0.1722998489425982, + "pass@2": 0.22345567595213742, + "pass@4": 0.271360043646357, + "pass@8": 0.3162899980235169, + "pass@16": 0.35667298671157666, + "pass@32": 0.38999713911741446, + "pass@64": 0.41914029524807356, + "pass@128": 0.44410876132930516 + }, + "20": { + "pass@1": 0.14937593843843844, + "pass@2": 0.20500889669196756, + "pass@4": 0.26037409597448957, + "pass@8": 0.3116453938615909, + "pass@16": 0.3592056707718412, + "pass@32": 0.40240030589143394, + "pass@64": 0.4403297922779381, + "pass@128": 0.4774774774774775 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..22a7185a4653698d79ecd3918e8c9b40acd2df61 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3e1bb0e4acce4f7b1b4bc119d541a9a4f574042e95eb4ddb66ce47ffe1a956b +size 1790508798 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a740e3305804a10a7eee739b93513c3cdb472cbc --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9841701388888889, + "avg_response_len": 198.90446006944444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9841701388888889, + "pass@2": 0.9883601405293072, + "pass@4": 0.9909663731200273, + "pass@8": 0.9927910715070025, + "pass@16": 0.9941717126607491, + "pass@32": 0.9952709968330081, + "pass@64": 0.9960523185714438, + "pass@128": 0.9965555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9364453125, + "pass@2": 0.952789985236221, + "pass@4": 0.964126053618298, + "pass@8": 0.9721539699401363, + "pass@16": 0.9784143815247518, + "pass@32": 0.983858956589895, + "pass@64": 0.9882498642963481, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975859375, + "pass@2": 0.9988038877952754, + "pass@4": 0.9995619341957254, + "pass@8": 0.9999119812009529, + "pass@16": 0.9999950451888476, + "pass@32": 0.9999999911142261, + "pass@64": 0.9999999999999999, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9946640625, + "pass@2": 0.9952380659448818, + "pass@4": 0.9958987766216721, + "pass@8": 0.9964952588157009, + "pass@16": 0.9969061862767266, + "pass@32": 0.9972137582666974, + "pass@64": 0.9974996944261582, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9793046875, + "pass@2": 0.984086491141732, + "pass@4": 0.9872383188038996, + "pass@8": 0.9895569036096646, + "pass@16": 0.9915759941233463, + "pass@32": 0.9931222502893143, + "pass@64": 0.9938557776673558, + "pass@128": 0.994 + }, + "6": { + "pass@1": 0.994671875, + "pass@2": 0.9960397391732284, + "pass@4": 0.9968090632733408, + "pass@8": 0.9973569597290509, + "pass@16": 0.9978164367809381, + "pass@32": 0.9981817789261483, + "pass@64": 0.9984985552627327, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9911171875, + "pass@2": 0.9935759104330709, + "pass@4": 0.9944777723097113, + "pass@8": 0.9948899979500961, + "pass@16": 0.9949953180757127, + "pass@32": 0.9949999964454285, + "pass@64": 0.995, + "pass@128": 0.995 + }, + "8": { + "pass@1": 0.9841015625, + "pass@2": 0.9892824803149609, + "pass@4": 0.9923239667229095, + "pass@8": 0.9944422962525519, + "pass@16": 0.9958772561628337, + "pass@32": 0.9967055239207104, + "pass@64": 0.9969830061520021, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.97965625, + "pass@2": 0.9854248277559056, + "pass@4": 0.9882614725346833, + "pass@8": 0.9903122760648695, + "pass@16": 0.9919647958135821, + "pass@32": 0.9933567159446538, + "pass@64": 0.9943839693383979, + "pass@128": 0.995 + } + }, + "per_op_accuracy": { + "10": 0.9364453125, + "2": 0.999984375, + "3": 0.9975859375, + "4": 0.9946640625, + "5": 0.9793046875, + "6": 0.994671875, + "7": 0.9911171875, + "8": 0.9841015625, + "9": 0.97965625 + }, + "per_op_avg_response_len": { + "10": 278.7620703125, + "2": 115.03321875, + "3": 144.0203359375, + "4": 159.345015625, + "5": 172.9409765625, + "6": 197.82378125, + "7": 222.5833203125, + "8": 241.9794453125, + "9": 257.6519765625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.7620703125, + "2": 115.03321875, + "3": 144.0203359375, + "4": 159.345015625, + "5": 172.9409765625, + "6": 197.82378125, + "7": 222.5833203125, + "8": 241.9794453125, + "9": 257.6519765625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 372868, + "answer_accuracy": 0.983799814251942, + "avg_response_len": 189.6478200987842, + "resp_tokens_sum": 71878041, + "pass_at_k": { + "pass@1": 0.983799814251942, + "pass@2": 0.9886851327759563, + "pass@4": 0.991599439412779, + "pass@8": 0.993486801364439, + "pass@16": 0.9948175037948571, + "pass@32": 0.9958147882342179, + "pass@64": 0.9964789094403068, + "pass@128": 0.9969604863221885 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9395510693215339, + "pass@2": 0.9560893694980606, + "pass@4": 0.9661865170614732, + "pass@8": 0.9718308736472469, + "pass@16": 0.9757989608364898, + "pass@32": 0.979932334585017, + "pass@64": 0.9840984873634936, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9960340214067278, + "pass@2": 0.9976123612415422, + "pass@4": 0.9989550615232728, + "pass@8": 0.9997536153738203, + "pass@16": 0.9999849525621903, + "pass@32": 0.9999999728267069, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9865966796875, + "pass@2": 0.9881970503198818, + "pass@4": 0.9902110206927258, + "pass@8": 0.9919773712990653, + "pass@16": 0.9930662071147704, + "pass@32": 0.9936367445834293, + "pass@64": 0.9937490450817442, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9731646825396826, + "pass@2": 0.9810359642544683, + "pass@4": 0.9860107715702207, + "pass@8": 0.9894212066960306, + "pass@16": 0.992812400217923, + "pass@32": 0.9956410092080746, + "pass@64": 0.9967580895903175, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9959300149700598, + "pass@2": 0.997761489815644, + "pass@4": 0.998781176398609, + "pass@8": 0.9995406021302704, + "pass@16": 0.999939161856703, + "pass@32": 0.999999382006006, + "pass@64": 0.999999999998189, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9942908653846154, + "pass@2": 0.9964044227274844, + "pass@4": 0.9969316950211105, + "pass@8": 0.9970380006710694, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9875801282051282, + "pass@2": 0.9916362242580252, + "pass@4": 0.9932182975325201, + "pass@8": 0.9943866487515717, + "pass@16": 0.9953653001527497, + "pass@32": 0.9962469963996953, + "pass@64": 0.9967505650677339, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9823283450704225, + "pass@2": 0.990102029499834, + "pass@4": 0.9944494404748702, + "pass@8": 0.9976051880920426, + "pass@16": 0.9994731157529941, + "pass@32": 0.9999746557434164, + "pass@64": 0.9999999850391206, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 384420, + "answer_accuracy": 0.9840371068152032, + "avg_response_len": 196.84911021461338, + "resp_tokens_sum": 76900286, + "pass_at_k": { + "pass@1": 0.9840371068152032, + "pass@2": 0.987596836913448, + "pass@4": 0.9898666126657985, + "pass@8": 0.991685232467629, + "pass@16": 0.9932960853859766, + "pass@32": 0.9945827812195723, + "pass@64": 0.995348019482976, + "pass@128": 0.995740498034076 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312452070552147, + "pass@2": 0.9476300963721564, + "pass@4": 0.9595703534565844, + "pass@8": 0.9693238635645908, + "pass@16": 0.977856514374984, + "pass@32": 0.9845534122942607, + "pass@64": 0.9888377362537187, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997466856060606, + "pass@2": 0.9989266434025291, + "pass@4": 0.9997181815341264, + "pass@8": 0.9999774434263569, + "pass@16": 0.999999896063691, + "pass@32": 0.9999999999996751, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9970183062130178, + "pass@2": 0.9970872839304851, + "pass@4": 0.997133875739645, + "pass@8": 0.9972263313609467, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9876358695652174, + "pass@2": 0.9890929904142418, + "pass@4": 0.9900907995196251, + "pass@8": 0.9913372622370229, + "pass@16": 0.9928546791547413, + "pass@32": 0.9939537238689515, + "pass@64": 0.9942000773644227, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9916278653295129, + "pass@2": 0.9936175208131218, + "pass@4": 0.9947991416903831, + "pass@8": 0.9955527181018352, + "pass@16": 0.9963087585123196, + "pass@32": 0.9969397860634448, + "pass@64": 0.9971305308405086, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9920971820809249, + "pass@2": 0.9939501217513995, + "pass@4": 0.9942145271039097, + "pass@8": 0.9942196518583029, + "pass@16": 0.9942196531791907, + "pass@32": 0.9942196531791907, + "pass@64": 0.9942196531791907, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9850352112676056, + "pass@2": 0.9895454419429967, + "pass@4": 0.9926431336047782, + "pass@8": 0.9950279092303314, + "pass@16": 0.9965571797743397, + "pass@32": 0.9971242473858734, + "pass@64": 0.9971829233370108, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.9771014074213775, + "pass@4": 0.9795268012905571, + "pass@8": 0.9816602761359167, + "pass@16": 0.9838594112624593, + "pass@32": 0.9863087065311279, + "pass@64": 0.9878548913312226, + "pass@128": 0.9880239520958084 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 376476, + "answer_accuracy": 0.984673167057248, + "avg_response_len": 210.18060292517575, + "resp_tokens_sum": 80359611, + "pass_at_k": { + "pass@1": 0.984673167057248, + "pass@2": 0.9888178909790193, + "pass@4": 0.9914625095825194, + "pass@8": 0.9932313007136643, + "pass@16": 0.9944262245102672, + "pass@32": 0.9954351306506262, + "pass@64": 0.9963490662297299, + "pass@128": 0.9969869434214932 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.954472470325538, + "pass@4": 0.9664742958436163, + "pass@8": 0.9752349977661584, + "pass@16": 0.9816039107908584, + "pass@32": 0.9871566649481952, + "pass@64": 0.9918787315266004, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999180029154519, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998857821637427, + "pass@2": 0.9999985610351338, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9765395220588236, + "pass@2": 0.9818325903195925, + "pass@4": 0.9854811762500273, + "pass@8": 0.9878760824371831, + "pass@16": 0.9891330110185733, + "pass@32": 0.9899449930999485, + "pass@64": 0.9908175081049403, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9968923874164783, + "pass@4": 0.9969440060134438, + "pass@8": 0.9970425867507886, + "pass@16": 0.9972397476340694, + "pass@32": 0.9976340694006309, + "pass@64": 0.998422712933754, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866495253164557, + "pass@2": 0.9901407386873317, + "pass@4": 0.9921412437171618, + "pass@8": 0.9933264372794367, + "pass@16": 0.9936560718228766, + "pass@32": 0.9936708748273051, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9798470345345346, + "pass@2": 0.9867968362062849, + "pass@4": 0.9911457821713727, + "pass@8": 0.9938701323282102, + "pass@16": 0.995632083169025, + "pass@32": 0.9966887484144756, + "pass@64": 0.9969876638625533, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9844252411575563, + "pass@2": 0.9890248784717828, + "pass@4": 0.9905786801770357, + "pass@8": 0.9912794278546566, + "pass@16": 0.9920990236643333, + "pass@32": 0.9933717208178267, + "pass@64": 0.9949853406587843, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.51487421875, + "avg_response_len": 344.78927421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.51487421875, + "pass@2": 0.5815188484251885, + "pass@4": 0.6360684912635839, + "pass@8": 0.6806829738724091, + "pass@16": 0.7178341898192142, + "pass@32": 0.7494815853319808, + "pass@64": 0.776536980971799, + "pass@128": 0.7996 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.925640625, + "pass@2": 0.9447103838582681, + "pass@4": 0.9564411323584547, + "pass@8": 0.9646504157861272, + "pass@16": 0.9708960810227444, + "pass@32": 0.97607004958443, + "pass@64": 0.9810039900029734, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.8783984375, + "pass@2": 0.9099769931102374, + "pass@4": 0.927241792369703, + "pass@8": 0.9386908128010287, + "pass@16": 0.9479472817852298, + "pass@32": 0.9557237468352948, + "pass@64": 0.9620965764842461, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.7519453125, + "pass@2": 0.8130600393700795, + "pass@4": 0.8539433999812523, + "pass@8": 0.883423069244311, + "pass@16": 0.9054874310796094, + "pass@32": 0.9232558502175315, + "pass@64": 0.937976785273383, + "pass@128": 0.95 + }, + "14": { + "pass@1": 0.6632265625, + "pass@2": 0.7364045275590557, + "pass@4": 0.7890315537120361, + "pass@8": 0.8288332737799572, + "pass@16": 0.8603049611826261, + "pass@32": 0.8845454470358377, + "pass@64": 0.9021116026582292, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.566015625, + "pass@2": 0.6557535679133867, + "pass@4": 0.7211395567116612, + "pass@8": 0.7693281313335598, + "pass@16": 0.8064710560834639, + "pass@32": 0.8362510141107569, + "pass@64": 0.8604156685709846, + "pass@128": 0.879 + }, + "16": { + "pass@1": 0.4100546875, + "pass@2": 0.5050537647637797, + "pass@4": 0.5852057425946746, + "pass@8": 0.6506464424731693, + "pass@16": 0.7040791788760784, + "pass@32": 0.748041786718314, + "pass@64": 0.784239203667722, + "pass@128": 0.814 + }, + "17": { + "pass@1": 0.317453125, + "pass@2": 0.40129195374015847, + "pass@4": 0.4737704152605926, + "pass@8": 0.5345350015861257, + "pass@16": 0.58646611994853, + "pass@32": 0.6307667230347177, + "pass@64": 0.6675096980254214, + "pass@128": 0.698 + }, + "18": { + "pass@1": 0.2634921875, + "pass@2": 0.34425898129921284, + "pass@4": 0.419257750937383, + "pass@8": 0.4830628607701628, + "pass@16": 0.5360171431951352, + "pass@32": 0.5822665379651201, + "pass@64": 0.6235374800029776, + "pass@128": 0.657 + }, + "19": { + "pass@1": 0.1925703125, + "pass@2": 0.25904921259842517, + "pass@4": 0.3231429097300335, + "pass@8": 0.38034797035988605, + "pass@16": 0.4307048828761059, + "pass@32": 0.4756808335714351, + "pass@64": 0.5175683058346909, + "pass@128": 0.561 + }, + "20": { + "pass@1": 0.1799453125, + "pass@2": 0.24562906003936977, + "pass@4": 0.3115106589801272, + "pass@8": 0.37331176058979887, + "pass@16": 0.42996776214248694, + "pass@32": 0.4822138642463598, + "pass@64": 0.5289104991972935, + "pass@128": 0.568 + } + }, + "per_op_accuracy": { + "11": 0.925640625, + "12": 0.8783984375, + "13": 0.7519453125, + "14": 0.6632265625, + "15": 0.566015625, + "16": 0.4100546875, + "17": 0.317453125, + "18": 0.2634921875, + "19": 0.1925703125, + "20": 0.1799453125 + }, + "per_op_avg_response_len": { + "11": 296.326625, + "12": 306.9516953125, + "13": 324.647546875, + "14": 344.3502890625, + "15": 355.2793046875, + "16": 354.87428125, + "17": 364.8006484375, + "18": 370.2595234375, + "19": 361.497015625, + "20": 368.9058125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.326625, + "12": 306.9516953125, + "13": 324.647546875, + "14": 344.3502890625, + "15": 355.2793046875, + "16": 354.87428125, + "17": 364.8006484375, + "18": 370.2595234375, + "19": 361.497015625, + "20": 368.9058125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 221881, + "answer_accuracy": 0.5151397659732541, + "avg_response_len": 326.9107378343239, + "resp_tokens_sum": 140806993, + "pass_at_k": { + "pass@1": 0.5151397659732541, + "pass@2": 0.5812074417053766, + "pass@4": 0.6345563089903533, + "pass@8": 0.6779527189630938, + "pass@16": 0.7146428098337451, + "pass@32": 0.746324950370175, + "pass@64": 0.7727007405105056, + "pass@128": 0.7931649331352154 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9224282932011332, + "pass@2": 0.943411088309429, + "pass@4": 0.9547086916897428, + "pass@8": 0.9609455199398679, + "pass@16": 0.9652854871074543, + "pass@32": 0.9700778900280648, + "pass@64": 0.9766359888655483, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8782000353107344, + "pass@2": 0.9146387322612222, + "pass@4": 0.9360131162100499, + "pass@8": 0.9497380889848458, + "pass@16": 0.9594810558399838, + "pass@32": 0.966746164791152, + "pass@64": 0.9717287111431125, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.7778458072100314, + "pass@2": 0.8323917631377581, + "pass@4": 0.8653471475352407, + "pass@8": 0.8874151171947741, + "pass@16": 0.9061765552344141, + "pass@32": 0.9251991379633109, + "pass@64": 0.9439211575782398, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.6514067367601246, + "pass@2": 0.7268309846199134, + "pass@4": 0.7818381183660449, + "pass@8": 0.8243974741890657, + "pass@16": 0.8587437898405418, + "pass@32": 0.8869336755282393, + "pass@64": 0.9090695958007434, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5669933164739884, + "pass@2": 0.6574636025897773, + "pass@4": 0.7201757048251913, + "pass@8": 0.7657689028200594, + "pass@16": 0.8026094244018224, + "pass@32": 0.8335503901204793, + "pass@64": 0.8579270169724725, + "pass@128": 0.8728323699421965 + }, + "16": { + "pass@1": 0.42280745967741934, + "pass@2": 0.5213693802387603, + "pass@4": 0.6024918958718871, + "pass@8": 0.6660506598462892, + "pass@16": 0.7175138650486194, + "pass@32": 0.7618016074084585, + "pass@64": 0.7986348355292889, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.323270411849711, + "pass@2": 0.40737364025306066, + "pass@4": 0.47887959371769284, + "pass@8": 0.5394925619262246, + "pass@16": 0.5928844080795413, + "pass@32": 0.6368722359763382, + "pass@64": 0.6691234797202088, + "pass@128": 0.6936416184971098 + }, + "18": { + "pass@1": 0.22564732142857144, + "pass@2": 0.3006295697412823, + "pass@4": 0.3731906971449996, + "pass@8": 0.4395313536549588, + "pass@16": 0.4993211002681893, + "pass@32": 0.5504900313154462, + "pass@64": 0.5914989196107283, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.1832301051051051, + "pass@2": 0.24577111560379272, + "pass@4": 0.305965981448265, + "pass@8": 0.3583828424824149, + "pass@16": 0.4022874096811754, + "pass@32": 0.44062328177746973, + "pass@64": 0.4767926961946683, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.1820335960960961, + "pass@2": 0.24922153945283845, + "pass@4": 0.3175866533012202, + "pass@8": 0.381078690664037, + "pass@16": 0.437386336319329, + "pass@32": 0.4881375222413864, + "pass@64": 0.530746167921435, + "pass@128": 0.5615615615615616 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 221005, + "answer_accuracy": 0.513411109872138, + "avg_response_len": 340.3289961529884, + "resp_tokens_sum": 146499381, + "pass_at_k": { + "pass@1": 0.513411109872138, + "pass@2": 0.5807073443986329, + "pass@4": 0.6360384066621481, + "pass@8": 0.6810963955607915, + "pass@16": 0.7181883986247586, + "pass@32": 0.7493872395127644, + "pass@64": 0.7764946006500014, + "pass@128": 0.8010704727921498 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9294521837349398, + "pass@2": 0.9492543253723558, + "pass@4": 0.9632092565989492, + "pass@8": 0.9745606085482055, + "pass@16": 0.9829470002547248, + "pass@32": 0.9877569971229867, + "pass@64": 0.9902069294682393, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8849676724137931, + "pass@2": 0.9097033668205269, + "pass@4": 0.9215010478705831, + "pass@8": 0.9284044720240515, + "pass@16": 0.9345690097180532, + "pass@32": 0.9411135687615974, + "pass@64": 0.9472845202813173, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7379858193277311, + "pass@2": 0.7976641936302079, + "pass@4": 0.8385840919044779, + "pass@8": 0.8712399569502159, + "pass@16": 0.8979985529612675, + "pass@32": 0.9185540345388505, + "pass@64": 0.9331883643143764, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6698876096491229, + "pass@2": 0.7393102897499655, + "pass@4": 0.7901910312197815, + "pass@8": 0.8302504396960447, + "pass@16": 0.8620678300466851, + "pass@32": 0.8853567958033306, + "pass@64": 0.9013693041268944, + "pass@128": 0.9122807017543859 + }, + "15": { + "pass@1": 0.563470643939394, + "pass@2": 0.6658662162968262, + "pass@4": 0.7395460959425525, + "pass@8": 0.7905438774062148, + "pass@16": 0.8268695671413484, + "pass@32": 0.8539699290321883, + "pass@64": 0.8762145726493178, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.4029780982905983, + "pass@2": 0.4960640436772324, + "pass@4": 0.5742025339781245, + "pass@8": 0.6380525027020472, + "pass@16": 0.690643820153264, + "pass@32": 0.7332660598943348, + "pass@64": 0.7685648755468092, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.31752232142857145, + "pass@2": 0.4047556640338432, + "pass@4": 0.4801637456420427, + "pass@8": 0.541317954329723, + "pass@16": 0.5903269161786768, + "pass@32": 0.6305932451179509, + "pass@64": 0.6648502775743499, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.2795955882352941, + "pass@2": 0.3619351406901343, + "pass@4": 0.436881981847857, + "pass@8": 0.49691001354059305, + "pass@16": 0.5430593127067409, + "pass@32": 0.5859046857139887, + "pass@64": 0.6308859898877225, + "pass@128": 0.6735294117647059 + }, + "19": { + "pass@1": 0.18138485863095238, + "pass@2": 0.24762468656261719, + "pass@4": 0.3143202247598515, + "pass@8": 0.3764716276443134, + "pass@16": 0.4336805519161678, + "pass@32": 0.48472837627745724, + "pass@64": 0.5295562201660091, + "pass@128": 0.5803571428571429 + }, + "20": { + "pass@1": 0.17220247005988024, + "pass@2": 0.2380696284596164, + "pass@4": 0.30321408420504314, + "pass@8": 0.3630226163803835, + "pass@16": 0.4181555338418773, + "pass@32": 0.47008713893128473, + "pass@64": 0.5196746561478678, + "pass@128": 0.5658682634730539 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 216153, + "answer_accuracy": 0.5161049243581907, + "avg_response_len": 367.7602980783924, + "resp_tokens_sum": 154023897, + "pass_at_k": { + "pass@1": 0.5161049243581907, + "pass@2": 0.5826731796151574, + "pass@4": 0.6376545755741436, + "pass@8": 0.6830659111070919, + "pass@16": 0.7207522183760504, + "pass@32": 0.7528249110155002, + "pass@64": 0.7805258178221538, + "pass@128": 0.8047066014669927 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9252232142857143, + "pass@2": 0.9413772497187851, + "pass@4": 0.9512491777813488, + "pass@8": 0.958357222886824, + "pass@16": 0.9644822221880784, + "pass@32": 0.9704674011418778, + "pass@64": 0.9761993185713631, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8722047018348624, + "pass@2": 0.9051972717859814, + "pass@4": 0.9233465289315899, + "pass@8": 0.9367660633783524, + "pass@16": 0.9485121526538749, + "pass@32": 0.9580439634993192, + "pass@64": 0.9661188096937128, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7418258101851852, + "pass@2": 0.8109905858121901, + "pass@4": 0.8596393182333685, + "pass@8": 0.8929166118146633, + "pass@16": 0.9130605757798124, + "pass@32": 0.9265232557927956, + "pass@64": 0.9374002776101592, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6677253338278932, + "pass@2": 0.7425746658800442, + "pass@4": 0.794706780532255, + "pass@8": 0.8316202794932324, + "pass@16": 0.8600029874416798, + "pass@32": 0.8814472196633035, + "pass@64": 0.8962372712011646, + "pass@128": 0.9080118694362018 + }, + "15": { + "pass@1": 0.5675636574074074, + "pass@2": 0.6436275001215127, + "pass@4": 0.703421454262661, + "pass@8": 0.7515204055980503, + "pass@16": 0.7898186360610788, + "pass@32": 0.8210879708285469, + "pass@64": 0.8469818263093642, + "pass@128": 0.8703703703703703 + }, + "16": { + "pass@1": 0.40571994837758113, + "pass@2": 0.4994418217081272, + "pass@4": 0.580791048519377, + "pass@8": 0.6495997329569347, + "pass@16": 0.7057047192838121, + "pass@32": 0.7507578212353399, + "pass@64": 0.7873042281318945, + "pass@128": 0.8200589970501475 + }, + "17": { + "pass@1": 0.3113234186746988, + "pass@2": 0.391594428896689, + "pass@4": 0.4622450293299178, + "pass@8": 0.5227897405586782, + "pass@16": 0.5760326739261331, + "pass@32": 0.6245720013823022, + "pass@64": 0.6684071827207451, + "pass@128": 0.7078313253012049 + }, + "18": { + "pass@1": 0.2885584677419355, + "pass@2": 0.37413123888747757, + "pass@4": 0.45193913905923033, + "pass@8": 0.5170241367326638, + "pass@16": 0.5697244896160558, + "pass@32": 0.614153012457931, + "pass@64": 0.6516503921851506, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.2133213746223565, + "pass@2": 0.2840046417441777, + "pass@4": 0.34937958425514043, + "pass@8": 0.406380715784752, + "pass@16": 0.4562734441342668, + "pass@32": 0.5017660009374981, + "pass@64": 0.5464212931301723, + "pass@128": 0.5891238670694864 + }, + "20": { + "pass@1": 0.18562312312312312, + "pass@2": 0.24961871320139006, + "pass@4": 0.3137561540730834, + "pass@8": 0.3758648730559356, + "pass@16": 0.43439688842331436, + "pass@32": 0.4884533481589473, + "pass@64": 0.5363384087869894, + "pass@128": 0.5765765765765766 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7371722861842105, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7371722861842105, + "pass@2": 0.7742331446850187, + "pass@4": 0.8041780142482032, + "pass@8": 0.8285236516993362, + "pass@16": 0.8487309111652063, + "pass@32": 0.8659081486745847, + "pass@64": 0.8805179303611016, + "pass@128": 0.8928947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9364453125, + "pass@2": 0.952789985236221, + "pass@4": 0.964126053618298, + "pass@8": 0.9721539699401363, + "pass@16": 0.9784143815247518, + "pass@32": 0.983858956589895, + "pass@64": 0.9882498642963481, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975859375, + "pass@2": 0.9988038877952754, + "pass@4": 0.9995619341957254, + "pass@8": 0.9999119812009529, + "pass@16": 0.9999950451888476, + "pass@32": 0.9999999911142261, + "pass@64": 0.9999999999999999, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9946640625, + "pass@2": 0.9952380659448818, + "pass@4": 0.9958987766216721, + "pass@8": 0.9964952588157009, + "pass@16": 0.9969061862767266, + "pass@32": 0.9972137582666974, + "pass@64": 0.9974996944261582, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9793046875, + "pass@2": 0.984086491141732, + "pass@4": 0.9872383188038996, + "pass@8": 0.9895569036096646, + "pass@16": 0.9915759941233463, + "pass@32": 0.9931222502893143, + "pass@64": 0.9938557776673558, + "pass@128": 0.994 + }, + "6": { + "pass@1": 0.994671875, + "pass@2": 0.9960397391732284, + "pass@4": 0.9968090632733408, + "pass@8": 0.9973569597290509, + "pass@16": 0.9978164367809381, + "pass@32": 0.9981817789261483, + "pass@64": 0.9984985552627327, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9911171875, + "pass@2": 0.9935759104330709, + "pass@4": 0.9944777723097113, + "pass@8": 0.9948899979500961, + "pass@16": 0.9949953180757127, + "pass@32": 0.9949999964454285, + "pass@64": 0.995, + "pass@128": 0.995 + }, + "8": { + "pass@1": 0.9841015625, + "pass@2": 0.9892824803149609, + "pass@4": 0.9923239667229095, + "pass@8": 0.9944422962525519, + "pass@16": 0.9958772561628337, + "pass@32": 0.9967055239207104, + "pass@64": 0.9969830061520021, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.97965625, + "pass@2": 0.9854248277559056, + "pass@4": 0.9882614725346833, + "pass@8": 0.9903122760648695, + "pass@16": 0.9919647958135821, + "pass@32": 0.9933567159446538, + "pass@64": 0.9943839693383979, + "pass@128": 0.995 + }, + "11": { + "pass@1": 0.925640625, + "pass@2": 0.9447103838582681, + "pass@4": 0.9564411323584547, + "pass@8": 0.9646504157861272, + "pass@16": 0.9708960810227444, + "pass@32": 0.97607004958443, + "pass@64": 0.9810039900029734, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.8783984375, + "pass@2": 0.9099769931102374, + "pass@4": 0.927241792369703, + "pass@8": 0.9386908128010287, + "pass@16": 0.9479472817852298, + "pass@32": 0.9557237468352948, + "pass@64": 0.9620965764842461, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.7519453125, + "pass@2": 0.8130600393700795, + "pass@4": 0.8539433999812523, + "pass@8": 0.883423069244311, + "pass@16": 0.9054874310796094, + "pass@32": 0.9232558502175315, + "pass@64": 0.937976785273383, + "pass@128": 0.95 + }, + "14": { + "pass@1": 0.6632265625, + "pass@2": 0.7364045275590557, + "pass@4": 0.7890315537120361, + "pass@8": 0.8288332737799572, + "pass@16": 0.8603049611826261, + "pass@32": 0.8845454470358377, + "pass@64": 0.9021116026582292, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.566015625, + "pass@2": 0.6557535679133867, + "pass@4": 0.7211395567116612, + "pass@8": 0.7693281313335598, + "pass@16": 0.8064710560834639, + "pass@32": 0.8362510141107569, + "pass@64": 0.8604156685709846, + "pass@128": 0.879 + }, + "16": { + "pass@1": 0.4100546875, + "pass@2": 0.5050537647637797, + "pass@4": 0.5852057425946746, + "pass@8": 0.6506464424731693, + "pass@16": 0.7040791788760784, + "pass@32": 0.748041786718314, + "pass@64": 0.784239203667722, + "pass@128": 0.814 + }, + "17": { + "pass@1": 0.317453125, + "pass@2": 0.40129195374015847, + "pass@4": 0.4737704152605926, + "pass@8": 0.5345350015861257, + "pass@16": 0.58646611994853, + "pass@32": 0.6307667230347177, + "pass@64": 0.6675096980254214, + "pass@128": 0.698 + }, + "18": { + "pass@1": 0.2634921875, + "pass@2": 0.34425898129921284, + "pass@4": 0.419257750937383, + "pass@8": 0.4830628607701628, + "pass@16": 0.5360171431951352, + "pass@32": 0.5822665379651201, + "pass@64": 0.6235374800029776, + "pass@128": 0.657 + }, + "19": { + "pass@1": 0.1925703125, + "pass@2": 0.25904921259842517, + "pass@4": 0.3231429097300335, + "pass@8": 0.38034797035988605, + "pass@16": 0.4307048828761059, + "pass@32": 0.4756808335714351, + "pass@64": 0.5175683058346909, + "pass@128": 0.561 + }, + "20": { + "pass@1": 0.1799453125, + "pass@2": 0.24562906003936977, + "pass@4": 0.3115106589801272, + "pass@8": 0.37331176058979887, + "pass@16": 0.42996776214248694, + "pass@32": 0.4822138642463598, + "pass@64": 0.5289104991972935, + "pass@128": 0.568 + } + }, + "per_op_accuracy": { + "10": 0.9364453125, + "2": 0.999984375, + "3": 0.9975859375, + "4": 0.9946640625, + "5": 0.9793046875, + "6": 0.994671875, + "7": 0.9911171875, + "8": 0.9841015625, + "9": 0.97965625, + "11": 0.925640625, + "12": 0.8783984375, + "13": 0.7519453125, + "14": 0.6632265625, + "15": 0.566015625, + "16": 0.4100546875, + "17": 0.317453125, + "18": 0.2634921875, + "19": 0.1925703125, + "20": 0.1799453125 + }, + "per_op_avg_response_len": { + "10": 278.7620703125, + "2": 115.03321875, + "3": 144.0203359375, + "4": 159.345015625, + "5": 172.9409765625, + "6": 197.82378125, + "7": 222.5833203125, + "8": 241.9794453125, + "9": 257.6519765625, + "11": 296.326625, + "12": 306.9516953125, + "13": 324.647546875, + "14": 344.3502890625, + "15": 355.2793046875, + "16": 354.87428125, + "17": 364.8006484375, + "18": 370.2595234375, + "19": 361.497015625, + "20": 368.9058125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.7620703125, + "2": 115.03321875, + "3": 144.0203359375, + "4": 159.345015625, + "5": 172.9409765625, + "6": 197.82378125, + "7": 222.5833203125, + "8": 241.9794453125, + "9": 257.6519765625, + "11": 296.326625, + "12": 306.9516953125, + "13": 324.647546875, + "14": 344.3502890625, + "15": 355.2793046875, + "16": 354.87428125, + "17": 364.8006484375, + "18": 370.2595234375, + "19": 361.497015625, + "20": 368.9058125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 594749, + "answer_accuracy": 0.7345046731742018, + "avg_response_len": 262.66231870455266, + "resp_tokens_sum": 212685034, + "pass_at_k": { + "pass@1": 0.7345046731742018, + "pass@2": 0.7719348276143125, + "pass@4": 0.8016768763600599, + "pass@8": 0.8256442172227113, + "pass@16": 0.8457836996249068, + "pass@32": 0.8631032320514007, + "pass@64": 0.8774442052909641, + "pass@128": 0.8885551691432184 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9395510693215339, + "pass@2": 0.9560893694980606, + "pass@4": 0.9661865170614732, + "pass@8": 0.9718308736472469, + "pass@16": 0.9757989608364898, + "pass@32": 0.979932334585017, + "pass@64": 0.9840984873634936, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9960340214067278, + "pass@2": 0.9976123612415422, + "pass@4": 0.9989550615232728, + "pass@8": 0.9997536153738203, + "pass@16": 0.9999849525621903, + "pass@32": 0.9999999728267069, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9865966796875, + "pass@2": 0.9881970503198818, + "pass@4": 0.9902110206927258, + "pass@8": 0.9919773712990653, + "pass@16": 0.9930662071147704, + "pass@32": 0.9936367445834293, + "pass@64": 0.9937490450817442, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9731646825396826, + "pass@2": 0.9810359642544683, + "pass@4": 0.9860107715702207, + "pass@8": 0.9894212066960306, + "pass@16": 0.992812400217923, + "pass@32": 0.9956410092080746, + "pass@64": 0.9967580895903175, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9959300149700598, + "pass@2": 0.997761489815644, + "pass@4": 0.998781176398609, + "pass@8": 0.9995406021302704, + "pass@16": 0.999939161856703, + "pass@32": 0.999999382006006, + "pass@64": 0.999999999998189, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9942908653846154, + "pass@2": 0.9964044227274844, + "pass@4": 0.9969316950211105, + "pass@8": 0.9970380006710694, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9875801282051282, + "pass@2": 0.9916362242580252, + "pass@4": 0.9932182975325201, + "pass@8": 0.9943866487515717, + "pass@16": 0.9953653001527497, + "pass@32": 0.9962469963996953, + "pass@64": 0.9967505650677339, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9823283450704225, + "pass@2": 0.990102029499834, + "pass@4": 0.9944494404748702, + "pass@8": 0.9976051880920426, + "pass@16": 0.9994731157529941, + "pass@32": 0.9999746557434164, + "pass@64": 0.9999999850391206, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9224282932011332, + "pass@2": 0.943411088309429, + "pass@4": 0.9547086916897428, + "pass@8": 0.9609455199398679, + "pass@16": 0.9652854871074543, + "pass@32": 0.9700778900280648, + "pass@64": 0.9766359888655483, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8782000353107344, + "pass@2": 0.9146387322612222, + "pass@4": 0.9360131162100499, + "pass@8": 0.9497380889848458, + "pass@16": 0.9594810558399838, + "pass@32": 0.966746164791152, + "pass@64": 0.9717287111431125, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.7778458072100314, + "pass@2": 0.8323917631377581, + "pass@4": 0.8653471475352407, + "pass@8": 0.8874151171947741, + "pass@16": 0.9061765552344141, + "pass@32": 0.9251991379633109, + "pass@64": 0.9439211575782398, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.6514067367601246, + "pass@2": 0.7268309846199134, + "pass@4": 0.7818381183660449, + "pass@8": 0.8243974741890657, + "pass@16": 0.8587437898405418, + "pass@32": 0.8869336755282393, + "pass@64": 0.9090695958007434, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5669933164739884, + "pass@2": 0.6574636025897773, + "pass@4": 0.7201757048251913, + "pass@8": 0.7657689028200594, + "pass@16": 0.8026094244018224, + "pass@32": 0.8335503901204793, + "pass@64": 0.8579270169724725, + "pass@128": 0.8728323699421965 + }, + "16": { + "pass@1": 0.42280745967741934, + "pass@2": 0.5213693802387603, + "pass@4": 0.6024918958718871, + "pass@8": 0.6660506598462892, + "pass@16": 0.7175138650486194, + "pass@32": 0.7618016074084585, + "pass@64": 0.7986348355292889, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.323270411849711, + "pass@2": 0.40737364025306066, + "pass@4": 0.47887959371769284, + "pass@8": 0.5394925619262246, + "pass@16": 0.5928844080795413, + "pass@32": 0.6368722359763382, + "pass@64": 0.6691234797202088, + "pass@128": 0.6936416184971098 + }, + "18": { + "pass@1": 0.22564732142857144, + "pass@2": 0.3006295697412823, + "pass@4": 0.3731906971449996, + "pass@8": 0.4395313536549588, + "pass@16": 0.4993211002681893, + "pass@32": 0.5504900313154462, + "pass@64": 0.5914989196107283, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.1832301051051051, + "pass@2": 0.24577111560379272, + "pass@4": 0.305965981448265, + "pass@8": 0.3583828424824149, + "pass@16": 0.4022874096811754, + "pass@32": 0.44062328177746973, + "pass@64": 0.4767926961946683, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.1820335960960961, + "pass@2": 0.24922153945283845, + "pass@4": 0.3175866533012202, + "pass@8": 0.381078690664037, + "pass@16": 0.437386336319329, + "pass@32": 0.4881375222413864, + "pass@64": 0.530746167921435, + "pass@128": 0.5615615615615616 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605425, + "answer_accuracy": 0.737316104832424, + "avg_response_len": 272.06701456547154, + "resp_tokens_sum": 223399667, + "pass_at_k": { + "pass@1": 0.737316104832424, + "pass@2": 0.774289063986345, + "pass@4": 0.804375691887887, + "pass@8": 0.8288621212411721, + "pass@16": 0.8490736145242556, + "pass@32": 0.8660414551463091, + "pass@64": 0.8806162895476283, + "pass@128": 0.8936866718628215 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312452070552147, + "pass@2": 0.9476300963721564, + "pass@4": 0.9595703534565844, + "pass@8": 0.9693238635645908, + "pass@16": 0.977856514374984, + "pass@32": 0.9845534122942607, + "pass@64": 0.9888377362537187, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997466856060606, + "pass@2": 0.9989266434025291, + "pass@4": 0.9997181815341264, + "pass@8": 0.9999774434263569, + "pass@16": 0.999999896063691, + "pass@32": 0.9999999999996751, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9970183062130178, + "pass@2": 0.9970872839304851, + "pass@4": 0.997133875739645, + "pass@8": 0.9972263313609467, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9876358695652174, + "pass@2": 0.9890929904142418, + "pass@4": 0.9900907995196251, + "pass@8": 0.9913372622370229, + "pass@16": 0.9928546791547413, + "pass@32": 0.9939537238689515, + "pass@64": 0.9942000773644227, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9916278653295129, + "pass@2": 0.9936175208131218, + "pass@4": 0.9947991416903831, + "pass@8": 0.9955527181018352, + "pass@16": 0.9963087585123196, + "pass@32": 0.9969397860634448, + "pass@64": 0.9971305308405086, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9920971820809249, + "pass@2": 0.9939501217513995, + "pass@4": 0.9942145271039097, + "pass@8": 0.9942196518583029, + "pass@16": 0.9942196531791907, + "pass@32": 0.9942196531791907, + "pass@64": 0.9942196531791907, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9850352112676056, + "pass@2": 0.9895454419429967, + "pass@4": 0.9926431336047782, + "pass@8": 0.9950279092303314, + "pass@16": 0.9965571797743397, + "pass@32": 0.9971242473858734, + "pass@64": 0.9971829233370108, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.9771014074213775, + "pass@4": 0.9795268012905571, + "pass@8": 0.9816602761359167, + "pass@16": 0.9838594112624593, + "pass@32": 0.9863087065311279, + "pass@64": 0.9878548913312226, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.9294521837349398, + "pass@2": 0.9492543253723558, + "pass@4": 0.9632092565989492, + "pass@8": 0.9745606085482055, + "pass@16": 0.9829470002547248, + "pass@32": 0.9877569971229867, + "pass@64": 0.9902069294682393, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8849676724137931, + "pass@2": 0.9097033668205269, + "pass@4": 0.9215010478705831, + "pass@8": 0.9284044720240515, + "pass@16": 0.9345690097180532, + "pass@32": 0.9411135687615974, + "pass@64": 0.9472845202813173, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7379858193277311, + "pass@2": 0.7976641936302079, + "pass@4": 0.8385840919044779, + "pass@8": 0.8712399569502159, + "pass@16": 0.8979985529612675, + "pass@32": 0.9185540345388505, + "pass@64": 0.9331883643143764, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6698876096491229, + "pass@2": 0.7393102897499655, + "pass@4": 0.7901910312197815, + "pass@8": 0.8302504396960447, + "pass@16": 0.8620678300466851, + "pass@32": 0.8853567958033306, + "pass@64": 0.9013693041268944, + "pass@128": 0.9122807017543859 + }, + "15": { + "pass@1": 0.563470643939394, + "pass@2": 0.6658662162968262, + "pass@4": 0.7395460959425525, + "pass@8": 0.7905438774062148, + "pass@16": 0.8268695671413484, + "pass@32": 0.8539699290321883, + "pass@64": 0.8762145726493178, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.4029780982905983, + "pass@2": 0.4960640436772324, + "pass@4": 0.5742025339781245, + "pass@8": 0.6380525027020472, + "pass@16": 0.690643820153264, + "pass@32": 0.7332660598943348, + "pass@64": 0.7685648755468092, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.31752232142857145, + "pass@2": 0.4047556640338432, + "pass@4": 0.4801637456420427, + "pass@8": 0.541317954329723, + "pass@16": 0.5903269161786768, + "pass@32": 0.6305932451179509, + "pass@64": 0.6648502775743499, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.2795955882352941, + "pass@2": 0.3619351406901343, + "pass@4": 0.436881981847857, + "pass@8": 0.49691001354059305, + "pass@16": 0.5430593127067409, + "pass@32": 0.5859046857139887, + "pass@64": 0.6308859898877225, + "pass@128": 0.6735294117647059 + }, + "19": { + "pass@1": 0.18138485863095238, + "pass@2": 0.24762468656261719, + "pass@4": 0.3143202247598515, + "pass@8": 0.3764716276443134, + "pass@16": 0.4336805519161678, + "pass@32": 0.48472837627745724, + "pass@64": 0.5295562201660091, + "pass@128": 0.5803571428571429 + }, + "20": { + "pass@1": 0.17220247005988024, + "pass@2": 0.2380696284596164, + "pass@4": 0.30321408420504314, + "pass@8": 0.3630226163803835, + "pass@16": 0.4181555338418773, + "pass@32": 0.47008713893128473, + "pass@64": 0.5196746561478678, + "pass@128": 0.5658682634730539 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 592629, + "answer_accuracy": 0.7397210516855728, + "avg_response_len": 292.5581013340789, + "resp_tokens_sum": 234383508, + "pass_at_k": { + "pass@1": 0.7397210516855728, + "pass@2": 0.7764987512470166, + "pass@4": 0.8065033212017154, + "pass@8": 0.8310870037344791, + "pass@16": 0.8513584264480969, + "pass@32": 0.8686064617504632, + "pass@64": 0.8835237476821095, + "pass@128": 0.8964690845182937 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.954472470325538, + "pass@4": 0.9664742958436163, + "pass@8": 0.9752349977661584, + "pass@16": 0.9816039107908584, + "pass@32": 0.9871566649481952, + "pass@64": 0.9918787315266004, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999180029154519, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998857821637427, + "pass@2": 0.9999985610351338, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9765395220588236, + "pass@2": 0.9818325903195925, + "pass@4": 0.9854811762500273, + "pass@8": 0.9878760824371831, + "pass@16": 0.9891330110185733, + "pass@32": 0.9899449930999485, + "pass@64": 0.9908175081049403, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9968923874164783, + "pass@4": 0.9969440060134438, + "pass@8": 0.9970425867507886, + "pass@16": 0.9972397476340694, + "pass@32": 0.9976340694006309, + "pass@64": 0.998422712933754, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866495253164557, + "pass@2": 0.9901407386873317, + "pass@4": 0.9921412437171618, + "pass@8": 0.9933264372794367, + "pass@16": 0.9936560718228766, + "pass@32": 0.9936708748273051, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9798470345345346, + "pass@2": 0.9867968362062849, + "pass@4": 0.9911457821713727, + "pass@8": 0.9938701323282102, + "pass@16": 0.995632083169025, + "pass@32": 0.9966887484144756, + "pass@64": 0.9969876638625533, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9844252411575563, + "pass@2": 0.9890248784717828, + "pass@4": 0.9905786801770357, + "pass@8": 0.9912794278546566, + "pass@16": 0.9920990236643333, + "pass@32": 0.9933717208178267, + "pass@64": 0.9949853406587843, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9252232142857143, + "pass@2": 0.9413772497187851, + "pass@4": 0.9512491777813488, + "pass@8": 0.958357222886824, + "pass@16": 0.9644822221880784, + "pass@32": 0.9704674011418778, + "pass@64": 0.9761993185713631, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8722047018348624, + "pass@2": 0.9051972717859814, + "pass@4": 0.9233465289315899, + "pass@8": 0.9367660633783524, + "pass@16": 0.9485121526538749, + "pass@32": 0.9580439634993192, + "pass@64": 0.9661188096937128, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7418258101851852, + "pass@2": 0.8109905858121901, + "pass@4": 0.8596393182333685, + "pass@8": 0.8929166118146633, + "pass@16": 0.9130605757798124, + "pass@32": 0.9265232557927956, + "pass@64": 0.9374002776101592, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6677253338278932, + "pass@2": 0.7425746658800442, + "pass@4": 0.794706780532255, + "pass@8": 0.8316202794932324, + "pass@16": 0.8600029874416798, + "pass@32": 0.8814472196633035, + "pass@64": 0.8962372712011646, + "pass@128": 0.9080118694362018 + }, + "15": { + "pass@1": 0.5675636574074074, + "pass@2": 0.6436275001215127, + "pass@4": 0.703421454262661, + "pass@8": 0.7515204055980503, + "pass@16": 0.7898186360610788, + "pass@32": 0.8210879708285469, + "pass@64": 0.8469818263093642, + "pass@128": 0.8703703703703703 + }, + "16": { + "pass@1": 0.40571994837758113, + "pass@2": 0.4994418217081272, + "pass@4": 0.580791048519377, + "pass@8": 0.6495997329569347, + "pass@16": 0.7057047192838121, + "pass@32": 0.7507578212353399, + "pass@64": 0.7873042281318945, + "pass@128": 0.8200589970501475 + }, + "17": { + "pass@1": 0.3113234186746988, + "pass@2": 0.391594428896689, + "pass@4": 0.4622450293299178, + "pass@8": 0.5227897405586782, + "pass@16": 0.5760326739261331, + "pass@32": 0.6245720013823022, + "pass@64": 0.6684071827207451, + "pass@128": 0.7078313253012049 + }, + "18": { + "pass@1": 0.2885584677419355, + "pass@2": 0.37413123888747757, + "pass@4": 0.45193913905923033, + "pass@8": 0.5170241367326638, + "pass@16": 0.5697244896160558, + "pass@32": 0.614153012457931, + "pass@64": 0.6516503921851506, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.2133213746223565, + "pass@2": 0.2840046417441777, + "pass@4": 0.34937958425514043, + "pass@8": 0.406380715784752, + "pass@16": 0.4562734441342668, + "pass@32": 0.5017660009374981, + "pass@64": 0.5464212931301723, + "pass@128": 0.5891238670694864 + }, + "20": { + "pass@1": 0.18562312312312312, + "pass@2": 0.24961871320139006, + "pass@4": 0.3137561540730834, + "pass@8": 0.3758648730559356, + "pass@16": 0.43439688842331436, + "pass@32": 0.4884533481589473, + "pass@64": 0.5363384087869894, + "pass@128": 0.5765765765765766 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1b456678d46e915c73cfb4cbc7379b3cadfd90f1 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d4ad0a237016288872bed26656147a0645435e39e197b26fd8d3f2ad93b3f91 +size 2626835475 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/summary.csv b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..0f4f1ee75730b54f4d9b96db9f7c4d588f922134 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op9-12,0.9841701388888889,0.51487421875,0.7371722861842105,0.0,0.0,0.0,198.90446006944444,344.78927421875,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/summary.json b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..32cea3ac06adc56d34f5ac71a091b74ae33e8a2a --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op9-12", + "id_acc": 0.9841701388888889, + "ood_acc": 0.51487421875, + "total_acc": 0.7371722861842105, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.90446006944444, + "ood_avg_resp_len": 344.78927421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 594749, + "answer_accuracy": 0.7345046731742018, + "avg_response_len": 262.66231870455266, + "resp_tokens_sum": 212685034, + "pass_at_k": { + "pass@1": 0.7345046731742018, + "pass@2": 0.7719348276143125, + "pass@4": 0.8016768763600599, + "pass@8": 0.8256442172227113, + "pass@16": 0.8457836996249068, + "pass@32": 0.8631032320514007, + "pass@64": 0.8774442052909641, + "pass@128": 0.8885551691432184 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9395510693215339, + "pass@2": 0.9560893694980606, + "pass@4": 0.9661865170614732, + "pass@8": 0.9718308736472469, + "pass@16": 0.9757989608364898, + "pass@32": 0.979932334585017, + "pass@64": 0.9840984873634936, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9960340214067278, + "pass@2": 0.9976123612415422, + "pass@4": 0.9989550615232728, + "pass@8": 0.9997536153738203, + "pass@16": 0.9999849525621903, + "pass@32": 0.9999999728267069, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9865966796875, + "pass@2": 0.9881970503198818, + "pass@4": 0.9902110206927258, + "pass@8": 0.9919773712990653, + "pass@16": 0.9930662071147704, + "pass@32": 0.9936367445834293, + "pass@64": 0.9937490450817442, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9731646825396826, + "pass@2": 0.9810359642544683, + "pass@4": 0.9860107715702207, + "pass@8": 0.9894212066960306, + "pass@16": 0.992812400217923, + "pass@32": 0.9956410092080746, + "pass@64": 0.9967580895903175, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9959300149700598, + "pass@2": 0.997761489815644, + "pass@4": 0.998781176398609, + "pass@8": 0.9995406021302704, + "pass@16": 0.999939161856703, + "pass@32": 0.999999382006006, + "pass@64": 0.999999999998189, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9942908653846154, + "pass@2": 0.9964044227274844, + "pass@4": 0.9969316950211105, + "pass@8": 0.9970380006710694, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9875801282051282, + "pass@2": 0.9916362242580252, + "pass@4": 0.9932182975325201, + "pass@8": 0.9943866487515717, + "pass@16": 0.9953653001527497, + "pass@32": 0.9962469963996953, + "pass@64": 0.9967505650677339, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9823283450704225, + "pass@2": 0.990102029499834, + "pass@4": 0.9944494404748702, + "pass@8": 0.9976051880920426, + "pass@16": 0.9994731157529941, + "pass@32": 0.9999746557434164, + "pass@64": 0.9999999850391206, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9224282932011332, + "pass@2": 0.943411088309429, + "pass@4": 0.9547086916897428, + "pass@8": 0.9609455199398679, + "pass@16": 0.9652854871074543, + "pass@32": 0.9700778900280648, + "pass@64": 0.9766359888655483, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8782000353107344, + "pass@2": 0.9146387322612222, + "pass@4": 0.9360131162100499, + "pass@8": 0.9497380889848458, + "pass@16": 0.9594810558399838, + "pass@32": 0.966746164791152, + "pass@64": 0.9717287111431125, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.7778458072100314, + "pass@2": 0.8323917631377581, + "pass@4": 0.8653471475352407, + "pass@8": 0.8874151171947741, + "pass@16": 0.9061765552344141, + "pass@32": 0.9251991379633109, + "pass@64": 0.9439211575782398, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.6514067367601246, + "pass@2": 0.7268309846199134, + "pass@4": 0.7818381183660449, + "pass@8": 0.8243974741890657, + "pass@16": 0.8587437898405418, + "pass@32": 0.8869336755282393, + "pass@64": 0.9090695958007434, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5669933164739884, + "pass@2": 0.6574636025897773, + "pass@4": 0.7201757048251913, + "pass@8": 0.7657689028200594, + "pass@16": 0.8026094244018224, + "pass@32": 0.8335503901204793, + "pass@64": 0.8579270169724725, + "pass@128": 0.8728323699421965 + }, + "16": { + "pass@1": 0.42280745967741934, + "pass@2": 0.5213693802387603, + "pass@4": 0.6024918958718871, + "pass@8": 0.6660506598462892, + "pass@16": 0.7175138650486194, + "pass@32": 0.7618016074084585, + "pass@64": 0.7986348355292889, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.323270411849711, + "pass@2": 0.40737364025306066, + "pass@4": 0.47887959371769284, + "pass@8": 0.5394925619262246, + "pass@16": 0.5928844080795413, + "pass@32": 0.6368722359763382, + "pass@64": 0.6691234797202088, + "pass@128": 0.6936416184971098 + }, + "18": { + "pass@1": 0.22564732142857144, + "pass@2": 0.3006295697412823, + "pass@4": 0.3731906971449996, + "pass@8": 0.4395313536549588, + "pass@16": 0.4993211002681893, + "pass@32": 0.5504900313154462, + "pass@64": 0.5914989196107283, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.1832301051051051, + "pass@2": 0.24577111560379272, + "pass@4": 0.305965981448265, + "pass@8": 0.3583828424824149, + "pass@16": 0.4022874096811754, + "pass@32": 0.44062328177746973, + "pass@64": 0.4767926961946683, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.1820335960960961, + "pass@2": 0.24922153945283845, + "pass@4": 0.3175866533012202, + "pass@8": 0.381078690664037, + "pass@16": 0.437386336319329, + "pass@32": 0.4881375222413864, + "pass@64": 0.530746167921435, + "pass@128": 0.5615615615615616 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605425, + "answer_accuracy": 0.737316104832424, + "avg_response_len": 272.06701456547154, + "resp_tokens_sum": 223399667, + "pass_at_k": { + "pass@1": 0.737316104832424, + "pass@2": 0.774289063986345, + "pass@4": 0.804375691887887, + "pass@8": 0.8288621212411721, + "pass@16": 0.8490736145242556, + "pass@32": 0.8660414551463091, + "pass@64": 0.8806162895476283, + "pass@128": 0.8936866718628215 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312452070552147, + "pass@2": 0.9476300963721564, + "pass@4": 0.9595703534565844, + "pass@8": 0.9693238635645908, + "pass@16": 0.977856514374984, + "pass@32": 0.9845534122942607, + "pass@64": 0.9888377362537187, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997466856060606, + "pass@2": 0.9989266434025291, + "pass@4": 0.9997181815341264, + "pass@8": 0.9999774434263569, + "pass@16": 0.999999896063691, + "pass@32": 0.9999999999996751, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9970183062130178, + "pass@2": 0.9970872839304851, + "pass@4": 0.997133875739645, + "pass@8": 0.9972263313609467, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9876358695652174, + "pass@2": 0.9890929904142418, + "pass@4": 0.9900907995196251, + "pass@8": 0.9913372622370229, + "pass@16": 0.9928546791547413, + "pass@32": 0.9939537238689515, + "pass@64": 0.9942000773644227, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9916278653295129, + "pass@2": 0.9936175208131218, + "pass@4": 0.9947991416903831, + "pass@8": 0.9955527181018352, + "pass@16": 0.9963087585123196, + "pass@32": 0.9969397860634448, + "pass@64": 0.9971305308405086, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9920971820809249, + "pass@2": 0.9939501217513995, + "pass@4": 0.9942145271039097, + "pass@8": 0.9942196518583029, + "pass@16": 0.9942196531791907, + "pass@32": 0.9942196531791907, + "pass@64": 0.9942196531791907, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9850352112676056, + "pass@2": 0.9895454419429967, + "pass@4": 0.9926431336047782, + "pass@8": 0.9950279092303314, + "pass@16": 0.9965571797743397, + "pass@32": 0.9971242473858734, + "pass@64": 0.9971829233370108, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.9771014074213775, + "pass@4": 0.9795268012905571, + "pass@8": 0.9816602761359167, + "pass@16": 0.9838594112624593, + "pass@32": 0.9863087065311279, + "pass@64": 0.9878548913312226, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.9294521837349398, + "pass@2": 0.9492543253723558, + "pass@4": 0.9632092565989492, + "pass@8": 0.9745606085482055, + "pass@16": 0.9829470002547248, + "pass@32": 0.9877569971229867, + "pass@64": 0.9902069294682393, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8849676724137931, + "pass@2": 0.9097033668205269, + "pass@4": 0.9215010478705831, + "pass@8": 0.9284044720240515, + "pass@16": 0.9345690097180532, + "pass@32": 0.9411135687615974, + "pass@64": 0.9472845202813173, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7379858193277311, + "pass@2": 0.7976641936302079, + "pass@4": 0.8385840919044779, + "pass@8": 0.8712399569502159, + "pass@16": 0.8979985529612675, + "pass@32": 0.9185540345388505, + "pass@64": 0.9331883643143764, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6698876096491229, + "pass@2": 0.7393102897499655, + "pass@4": 0.7901910312197815, + "pass@8": 0.8302504396960447, + "pass@16": 0.8620678300466851, + "pass@32": 0.8853567958033306, + "pass@64": 0.9013693041268944, + "pass@128": 0.9122807017543859 + }, + "15": { + "pass@1": 0.563470643939394, + "pass@2": 0.6658662162968262, + "pass@4": 0.7395460959425525, + "pass@8": 0.7905438774062148, + "pass@16": 0.8268695671413484, + "pass@32": 0.8539699290321883, + "pass@64": 0.8762145726493178, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.4029780982905983, + "pass@2": 0.4960640436772324, + "pass@4": 0.5742025339781245, + "pass@8": 0.6380525027020472, + "pass@16": 0.690643820153264, + "pass@32": 0.7332660598943348, + "pass@64": 0.7685648755468092, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.31752232142857145, + "pass@2": 0.4047556640338432, + "pass@4": 0.4801637456420427, + "pass@8": 0.541317954329723, + "pass@16": 0.5903269161786768, + "pass@32": 0.6305932451179509, + "pass@64": 0.6648502775743499, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.2795955882352941, + "pass@2": 0.3619351406901343, + "pass@4": 0.436881981847857, + "pass@8": 0.49691001354059305, + "pass@16": 0.5430593127067409, + "pass@32": 0.5859046857139887, + "pass@64": 0.6308859898877225, + "pass@128": 0.6735294117647059 + }, + "19": { + "pass@1": 0.18138485863095238, + "pass@2": 0.24762468656261719, + "pass@4": 0.3143202247598515, + "pass@8": 0.3764716276443134, + "pass@16": 0.4336805519161678, + "pass@32": 0.48472837627745724, + "pass@64": 0.5295562201660091, + "pass@128": 0.5803571428571429 + }, + "20": { + "pass@1": 0.17220247005988024, + "pass@2": 0.2380696284596164, + "pass@4": 0.30321408420504314, + "pass@8": 0.3630226163803835, + "pass@16": 0.4181555338418773, + "pass@32": 0.47008713893128473, + "pass@64": 0.5196746561478678, + "pass@128": 0.5658682634730539 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 592629, + "answer_accuracy": 0.7397210516855728, + "avg_response_len": 292.5581013340789, + "resp_tokens_sum": 234383508, + "pass_at_k": { + "pass@1": 0.7397210516855728, + "pass@2": 0.7764987512470166, + "pass@4": 0.8065033212017154, + "pass@8": 0.8310870037344791, + "pass@16": 0.8513584264480969, + "pass@32": 0.8686064617504632, + "pass@64": 0.8835237476821095, + "pass@128": 0.8964690845182937 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.954472470325538, + "pass@4": 0.9664742958436163, + "pass@8": 0.9752349977661584, + "pass@16": 0.9816039107908584, + "pass@32": 0.9871566649481952, + "pass@64": 0.9918787315266004, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999180029154519, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998857821637427, + "pass@2": 0.9999985610351338, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9765395220588236, + "pass@2": 0.9818325903195925, + "pass@4": 0.9854811762500273, + "pass@8": 0.9878760824371831, + "pass@16": 0.9891330110185733, + "pass@32": 0.9899449930999485, + "pass@64": 0.9908175081049403, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9968923874164783, + "pass@4": 0.9969440060134438, + "pass@8": 0.9970425867507886, + "pass@16": 0.9972397476340694, + "pass@32": 0.9976340694006309, + "pass@64": 0.998422712933754, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866495253164557, + "pass@2": 0.9901407386873317, + "pass@4": 0.9921412437171618, + "pass@8": 0.9933264372794367, + "pass@16": 0.9936560718228766, + "pass@32": 0.9936708748273051, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9798470345345346, + "pass@2": 0.9867968362062849, + "pass@4": 0.9911457821713727, + "pass@8": 0.9938701323282102, + "pass@16": 0.995632083169025, + "pass@32": 0.9966887484144756, + "pass@64": 0.9969876638625533, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9844252411575563, + "pass@2": 0.9890248784717828, + "pass@4": 0.9905786801770357, + "pass@8": 0.9912794278546566, + "pass@16": 0.9920990236643333, + "pass@32": 0.9933717208178267, + "pass@64": 0.9949853406587843, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9252232142857143, + "pass@2": 0.9413772497187851, + "pass@4": 0.9512491777813488, + "pass@8": 0.958357222886824, + "pass@16": 0.9644822221880784, + "pass@32": 0.9704674011418778, + "pass@64": 0.9761993185713631, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8722047018348624, + "pass@2": 0.9051972717859814, + "pass@4": 0.9233465289315899, + "pass@8": 0.9367660633783524, + "pass@16": 0.9485121526538749, + "pass@32": 0.9580439634993192, + "pass@64": 0.9661188096937128, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7418258101851852, + "pass@2": 0.8109905858121901, + "pass@4": 0.8596393182333685, + "pass@8": 0.8929166118146633, + "pass@16": 0.9130605757798124, + "pass@32": 0.9265232557927956, + "pass@64": 0.9374002776101592, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6677253338278932, + "pass@2": 0.7425746658800442, + "pass@4": 0.794706780532255, + "pass@8": 0.8316202794932324, + "pass@16": 0.8600029874416798, + "pass@32": 0.8814472196633035, + "pass@64": 0.8962372712011646, + "pass@128": 0.9080118694362018 + }, + "15": { + "pass@1": 0.5675636574074074, + "pass@2": 0.6436275001215127, + "pass@4": 0.703421454262661, + "pass@8": 0.7515204055980503, + "pass@16": 0.7898186360610788, + "pass@32": 0.8210879708285469, + "pass@64": 0.8469818263093642, + "pass@128": 0.8703703703703703 + }, + "16": { + "pass@1": 0.40571994837758113, + "pass@2": 0.4994418217081272, + "pass@4": 0.580791048519377, + "pass@8": 0.6495997329569347, + "pass@16": 0.7057047192838121, + "pass@32": 0.7507578212353399, + "pass@64": 0.7873042281318945, + "pass@128": 0.8200589970501475 + }, + "17": { + "pass@1": 0.3113234186746988, + "pass@2": 0.391594428896689, + "pass@4": 0.4622450293299178, + "pass@8": 0.5227897405586782, + "pass@16": 0.5760326739261331, + "pass@32": 0.6245720013823022, + "pass@64": 0.6684071827207451, + "pass@128": 0.7078313253012049 + }, + "18": { + "pass@1": 0.2885584677419355, + "pass@2": 0.37413123888747757, + "pass@4": 0.45193913905923033, + "pass@8": 0.5170241367326638, + "pass@16": 0.5697244896160558, + "pass@32": 0.614153012457931, + "pass@64": 0.6516503921851506, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.2133213746223565, + "pass@2": 0.2840046417441777, + "pass@4": 0.34937958425514043, + "pass@8": 0.406380715784752, + "pass@16": 0.4562734441342668, + "pass@32": 0.5017660009374981, + "pass@64": 0.5464212931301723, + "pass@128": 0.5891238670694864 + }, + "20": { + "pass@1": 0.18562312312312312, + "pass@2": 0.24961871320139006, + "pass@4": 0.3137561540730834, + "pass@8": 0.3758648730559356, + "pass@16": 0.43439688842331436, + "pass@32": 0.4884533481589473, + "pass@64": 0.5363384087869894, + "pass@128": 0.5765765765765766 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d25e1d2f9a1447f47bef1ab2b26c41b06628d197 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da58b952ae3fd6b4e43a989622a04f591a88cf3a0a906da994baa0987df8d747 +size 1789216551 diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..89beb8a51b52020272340a410dd39a72396c8056 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op7-10", + "id": { + "answer_accuracy": 0.9941623263888889, + "avg_response_len": 200.36135069444444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9941623263888889, + "pass@2": 0.995739679024497, + "pass@4": 0.9968566256717913, + "pass@8": 0.9977157160029017, + "pass@16": 0.9983953345383363, + "pass@32": 0.9989066885032427, + "pass@64": 0.9992577350637216, + "pass@128": 0.9994444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.958484375, + "pass@2": 0.9682966289370083, + "pass@4": 0.9761080770528682, + "pass@8": 0.9826235127203949, + "pass@16": 0.9877163443569874, + "pass@32": 0.991429265701168, + "pass@64": 0.9938933039035253, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998671875, + "pass@2": 0.9999927411417323, + "pass@4": 0.9999999690663667, + "pass@8": 0.9999999999998846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997265625, + "pass@2": 0.9999621062992126, + "pass@4": 0.9999991390138733, + "pass@8": 0.9999999996569364, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999609375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.99578125, + "pass@2": 0.9979758858267715, + "pass@4": 0.9988914759092613, + "pass@8": 0.9990593780831115, + "pass@16": 0.9991249987361553, + "pass@32": 0.99925, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993640625, + "pass@2": 0.9954298720472442, + "pass@4": 0.9967109700037493, + "pass@8": 0.9977585535657889, + "pass@16": 0.9987166677518824, + "pass@32": 0.9994809308280161, + "pass@64": 0.9999263116699708, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.958484375, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9998671875, + "6": 0.9997265625, + "7": 0.9999609375, + "8": 0.99578125, + "9": 0.993640625 + }, + "per_op_avg_response_len": { + "10": 281.5871796875, + "2": 115.7971171875, + "3": 144.9716328125, + "4": 160.1594765625, + "5": 173.0815546875, + "6": 199.3073515625, + "7": 224.3540546875, + "8": 244.032203125, + "9": 259.9615859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.5871796875, + "2": 115.7971171875, + "3": 144.9716328125, + "4": 160.1594765625, + "5": 173.0815546875, + "6": 199.3073515625, + "7": 224.3540546875, + "8": 244.032203125, + "9": 259.9615859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376965, + "answer_accuracy": 0.9946096124620061, + "avg_response_len": 190.89631089581223, + "resp_tokens_sum": 72351229, + "pass_at_k": { + "pass@1": 0.9946096124620061, + "pass@2": 0.9963618660034518, + "pass@4": 0.9973578219452558, + "pass@8": 0.9979648863040437, + "pass@16": 0.998477538397376, + "pass@32": 0.9989086437216105, + "pass@64": 0.9992113314189143, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9631959808259587, + "pass@2": 0.9731588100713074, + "pass@4": 0.9796233927617454, + "pass@8": 0.9842158548331358, + "pass@16": 0.9880032489114736, + "pass@32": 0.9909717471897269, + "pass@64": 0.9931521416822178, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994620134730539, + "pass@2": 0.9999068054835212, + "pass@4": 0.9999975148128939, + "pass@8": 0.9999999989732089, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956931089743589, + "pass@2": 0.9986860867151223, + "pass@4": 0.9998419906045398, + "pass@8": 0.9999969647115807, + "pass@16": 0.9999999989745152, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945642605633803, + "pass@2": 0.9965291255406454, + "pass@4": 0.9975614355423881, + "pass@8": 0.9981008476359892, + "pass@16": 0.9987574369960317, + "pass@32": 0.9995185120066794, + "pass@64": 0.999961060003192, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388156, + "answer_accuracy": 0.9936005078636959, + "avg_response_len": 198.390765788827, + "resp_tokens_sum": 77502543, + "pass_at_k": { + "pass@1": 0.9936005078636959, + "pass@2": 0.9951880253428758, + "pass@4": 0.9963414133767354, + "pass@8": 0.9973067203756558, + "pass@16": 0.9981106488861698, + "pass@32": 0.9987796211951983, + "pass@64": 0.9992922173957688, + "pass@128": 0.9996723460026212 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954011694785276, + "pass@2": 0.9650013133423504, + "pass@4": 0.9737770710984742, + "pass@8": 0.9814918018455097, + "pass@16": 0.9874433942451809, + "pass@32": 0.9919418373382088, + "pass@64": 0.9950911414749884, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999773550724638, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997537607449857, + "pass@2": 0.9999806110822824, + "pass@4": 0.9999999113649476, + "pass@8": 0.9999999999996694, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999322615606936, + "pass@2": 0.9999996444176414, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9950704225352113, + "pass@2": 0.9965544249750472, + "pass@4": 0.9971638872605713, + "pass@8": 0.9973558977814235, + "pass@16": 0.9975352095202358, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992000374251497, + "pass@2": 0.9938727368098449, + "pass@4": 0.9951782018078579, + "pass@8": 0.9962647888400472, + "pass@16": 0.9976112410089141, + "pass@32": 0.9989591763936809, + "pass@64": 0.9998207645839522, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380154, + "answer_accuracy": 0.9942929779042518, + "avg_response_len": 211.7574698694342, + "resp_tokens_sum": 80962504, + "pass_at_k": { + "pass@1": 0.9942929779042518, + "pass@2": 0.9956865659985923, + "pass@4": 0.9968862158153398, + "pass@8": 0.9978866103425981, + "pass@16": 0.9986047268998346, + "pass@32": 0.9990345827190323, + "pass@64": 0.9992685020924689, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9580690298507463, + "pass@2": 0.9665831766365026, + "pass@4": 0.9748191693389073, + "pass@8": 0.9821134642696296, + "pass@16": 0.9876916312002068, + "pass@32": 0.9913934460644604, + "pass@64": 0.9934776589624096, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999790122742009, + "pass@4": 0.9999999090187256, + "pass@8": 0.9999999999996606, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999753548895899, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9966216216216216, + "pass@2": 0.9988258435600955, + "pass@4": 0.999842627216643, + "pass@8": 0.9999969410831621, + "pass@16": 0.9999999990282966, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943478697749196, + "pass@2": 0.9958473903081245, + "pass@4": 0.9973863054095731, + "pass@8": 0.9989720681107301, + "pass@16": 0.9998573090717485, + "pass@32": 0.999998373473169, + "pass@64": 0.9999999999929186, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4831703125, + "avg_response_len": 336.90805078125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4831703125, + "pass@2": 0.5480781003936945, + "pass@4": 0.6019056518278872, + "pass@8": 0.6465263062092536, + "pass@16": 0.6840622030815707, + "pass@32": 0.7164242896421489, + "pass@64": 0.7449707579841154, + "pass@128": 0.77 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9446015625, + "pass@2": 0.9606361958661419, + "pass@4": 0.9706209278215225, + "pass@8": 0.9770803766579949, + "pass@16": 0.9812540369298193, + "pass@32": 0.9842091557330336, + "pass@64": 0.9862806773451164, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8609765625, + "pass@2": 0.8986541584645682, + "pass@4": 0.9198446199850016, + "pass@8": 0.933813023195648, + "pass@16": 0.9433933345589797, + "pass@32": 0.9498562494029792, + "pass@64": 0.9550037480484834, + "pass@128": 0.96 + }, + "13": { + "pass@1": 0.7124296875, + "pass@2": 0.7845658218503946, + "pass@4": 0.831534217754031, + "pass@8": 0.862838242321, + "pass@16": 0.8858632772953222, + "pass@32": 0.9047596826854089, + "pass@64": 0.9210772857822727, + "pass@128": 0.935 + }, + "14": { + "pass@1": 0.6589375, + "pass@2": 0.7366493602362207, + "pass@4": 0.7909602792463437, + "pass@8": 0.8290085420254784, + "pass@16": 0.8571618068972121, + "pass@32": 0.8796693898656633, + "pass@64": 0.8985783168716919, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.5503046875, + "pass@2": 0.647400959645669, + "pass@4": 0.7197840649606297, + "pass@8": 0.7744435087875599, + "pass@16": 0.817383191828716, + "pass@32": 0.8523338002808312, + "pass@64": 0.8800684135832626, + "pass@128": 0.9 + }, + "16": { + "pass@1": 0.3044375, + "pass@2": 0.3845978100393697, + "pass@4": 0.4561933964191972, + "pass@8": 0.5182463992333959, + "pass@16": 0.571895842343343, + "pass@32": 0.617956005780948, + "pass@64": 0.6595758910037973, + "pass@128": 0.698 + }, + "17": { + "pass@1": 0.254734375, + "pass@2": 0.3320616387795277, + "pass@4": 0.40505249887514033, + "pass@8": 0.4699472794329776, + "pass@16": 0.5256080803835056, + "pass@32": 0.5731372708577546, + "pass@64": 0.6170483351733107, + "pass@128": 0.662 + }, + "18": { + "pass@1": 0.2149375, + "pass@2": 0.28408624507874014, + "pass@4": 0.3513472681852263, + "pass@8": 0.41372310769530835, + "pass@16": 0.4717159005894626, + "pass@32": 0.5261657332909438, + "pass@64": 0.5750761510177632, + "pass@128": 0.615 + }, + "19": { + "pass@1": 0.173078125, + "pass@2": 0.23374274114173227, + "pass@4": 0.29328206955380576, + "pass@8": 0.3477914039665271, + "pass@16": 0.3970700170662132, + "pass@32": 0.44177308812314764, + "pass@64": 0.4800497528756157, + "pass@128": 0.511 + }, + "20": { + "pass@1": 0.157265625, + "pass@2": 0.21838607283464567, + "pass@4": 0.2804371754780648, + "pass@8": 0.3383711787766657, + "pass@16": 0.38927654292301656, + "pass@32": 0.4343825204007902, + "pass@64": 0.4769490081397651, + "pass@128": 0.516 + } + }, + "per_op_accuracy": { + "11": 0.9446015625, + "12": 0.8609765625, + "13": 0.7124296875, + "14": 0.6589375, + "15": 0.5503046875, + "16": 0.3044375, + "17": 0.254734375, + "18": 0.2149375, + "19": 0.173078125, + "20": 0.157265625 + }, + "per_op_avg_response_len": { + "11": 299.2658984375, + "12": 307.45903125, + "13": 322.2324453125, + "14": 345.3673515625, + "15": 356.779328125, + "16": 346.7696484375, + "17": 351.8356640625, + "18": 354.310234375, + "19": 338.7199921875, + "20": 346.3409140625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.2658984375, + "12": 307.45903125, + "13": 322.2324453125, + "14": 345.3673515625, + "15": 356.779328125, + "16": 346.7696484375, + "17": 351.8356640625, + "18": 354.310234375, + "19": 338.7199921875, + "20": 346.3409140625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 210427, + "answer_accuracy": 0.4885470839524517, + "avg_response_len": 320.2020872028232, + "resp_tokens_sum": 137917443, + "pass_at_k": { + "pass@1": 0.4885470839524517, + "pass@2": 0.5521766520223261, + "pass@4": 0.604603933609341, + "pass@8": 0.6478037469664155, + "pass@16": 0.684282598525169, + "pass@32": 0.7157845537255326, + "pass@64": 0.7434147486418271, + "pass@128": 0.7673105497771174 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9376991855524079, + "pass@2": 0.9547899333050791, + "pass@4": 0.9666048129429994, + "pass@8": 0.9747402292502628, + "pass@16": 0.9798547128476756, + "pass@32": 0.9834549289935417, + "pass@64": 0.9858023195208817, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8657530014124294, + "pass@2": 0.9054356983184302, + "pass@4": 0.9276475596800396, + "pass@8": 0.9411308026374295, + "pass@16": 0.9485681266608572, + "pass@32": 0.9527450938166466, + "pass@64": 0.9565891063317338, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7410854231974922, + "pass@2": 0.807538629575692, + "pass@4": 0.8486559937257053, + "pass@8": 0.8765056311963663, + "pass@16": 0.8984341628770319, + "pass@32": 0.9170324322596793, + "pass@64": 0.9311435811665233, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6596816588785047, + "pass@2": 0.7336943023769223, + "pass@4": 0.7882888003485545, + "pass@8": 0.8288986917843586, + "pass@16": 0.8592432109531271, + "pass@32": 0.882945852339974, + "pass@64": 0.9017789483871335, + "pass@128": 0.9158878504672897 + }, + "15": { + "pass@1": 0.5488168352601156, + "pass@2": 0.6472950138819349, + "pass@4": 0.7241442177025557, + "pass@8": 0.7831827839828602, + "pass@16": 0.8300831095008099, + "pass@32": 0.867248365688248, + "pass@64": 0.8934960977796641, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.310710685483871, + "pass@2": 0.3919696151892304, + "pass@4": 0.45937909666533605, + "pass@8": 0.5143017852386241, + "pass@16": 0.5637792315815162, + "pass@32": 0.6079851621930701, + "pass@64": 0.6491849484380694, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.25593840317919075, + "pass@2": 0.33026311672204245, + "pass@4": 0.3988996206579669, + "pass@8": 0.4607312429836123, + "pass@16": 0.514943009962257, + "pass@32": 0.561849951338928, + "pass@64": 0.6068011159502685, + "pass@128": 0.653179190751445 + }, + "18": { + "pass@1": 0.19513392857142858, + "pass@2": 0.2628627671541054, + "pass@4": 0.3290206976806473, + "pass@8": 0.38876375862284673, + "pass@16": 0.4435721695894926, + "pass@32": 0.4966436629883193, + "pass@64": 0.5472989033830987, + "pass@128": 0.5914285714285714 + }, + "19": { + "pass@1": 0.1796875, + "pass@2": 0.23905499988177165, + "pass@4": 0.2972096981683596, + "pass@8": 0.3521358209569636, + "pass@16": 0.4022299045017779, + "pass@32": 0.44642726263310534, + "pass@64": 0.4824560579294367, + "pass@128": 0.5045045045045045 + }, + "20": { + "pass@1": 0.1664085960960961, + "pass@2": 0.22833537277434907, + "pass@4": 0.28835667529171466, + "pass@8": 0.3416753045571554, + "pass@16": 0.38789227690805683, + "pass@32": 0.42881599752146604, + "pass@64": 0.4681722777240641, + "pass@128": 0.5075075075075075 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 205627, + "answer_accuracy": 0.4776868681236991, + "avg_response_len": 332.49271251486766, + "resp_tokens_sum": 143126143, + "pass_at_k": { + "pass@1": 0.4776868681236991, + "pass@2": 0.5429083226215827, + "pass@4": 0.5967457404131357, + "pass@8": 0.6413668212762412, + "pass@16": 0.6792030230839539, + "pass@32": 0.7125657949932793, + "pass@64": 0.7423528277886029, + "pass@128": 0.7689562890276539 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.949195218373494, + "pass@2": 0.962625847879708, + "pass@4": 0.9700664567719697, + "pass@8": 0.9747850987990536, + "pass@16": 0.9778305768375364, + "pass@32": 0.9800141809222661, + "pass@64": 0.981568883804507, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8523217084639498, + "pass@2": 0.883371926912349, + "pass@4": 0.9001900677383976, + "pass@8": 0.9134272201943227, + "pass@16": 0.9246892536945397, + "pass@32": 0.9329112526745617, + "pass@64": 0.9386328551278152, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.6943496148459384, + "pass@2": 0.7749250782990365, + "pass@4": 0.8268435101074553, + "pass@8": 0.8589385187033101, + "pass@16": 0.8807564765425908, + "pass@32": 0.8982965502713754, + "pass@64": 0.9146222978342695, + "pass@128": 0.9299719887955182 + }, + "14": { + "pass@1": 0.643937317251462, + "pass@2": 0.7241691416862364, + "pass@4": 0.7795188065855806, + "pass@8": 0.8185037908332939, + "pass@16": 0.8484280510819681, + "pass@32": 0.872423187936195, + "pass@64": 0.8915716940190292, + "pass@128": 0.9064327485380117 + }, + "15": { + "pass@1": 0.5625, + "pass@2": 0.6625883589835366, + "pass@4": 0.7347707221256433, + "pass@8": 0.7859226102348986, + "pass@16": 0.8225864124521411, + "pass@32": 0.8525939224935586, + "pass@64": 0.8795478506520482, + "pass@128": 0.9 + }, + "16": { + "pass@1": 0.3109642094017094, + "pass@2": 0.3854944814590484, + "pass@4": 0.4524724684521269, + "pass@8": 0.5122811532302066, + "pass@16": 0.5644623158256834, + "pass@32": 0.610120655316517, + "pass@64": 0.6507670832107987, + "pass@128": 0.6866096866096866 + }, + "17": { + "pass@1": 0.26657123447204967, + "pass@2": 0.3530736477233822, + "pass@4": 0.43370805211848495, + "pass@8": 0.5006082372902344, + "pass@16": 0.5540322646582369, + "pass@32": 0.5996876878591957, + "pass@64": 0.6425915232100543, + "pass@128": 0.6894409937888198 + }, + "18": { + "pass@1": 0.20723805147058824, + "pass@2": 0.2790444795044002, + "pass@4": 0.34853543720637853, + "pass@8": 0.41392390293806336, + "pass@16": 0.47702879320906416, + "pass@32": 0.5375902449727734, + "pass@64": 0.5894177166706436, + "pass@128": 0.6294117647058823 + }, + "19": { + "pass@1": 0.15404110863095238, + "pass@2": 0.20788990731627285, + "pass@4": 0.2633022258936383, + "pass@8": 0.317408618000936, + "pass@16": 0.37060408335097106, + "pass@32": 0.422477599367868, + "pass@64": 0.4701716838210624, + "pass@128": 0.5148809523809523 + }, + "20": { + "pass@1": 0.14520958083832336, + "pass@2": 0.20275627386958367, + "pass@4": 0.2632841339256246, + "pass@8": 0.32212848817550394, + "pass@16": 0.3750321787903876, + "pass@32": 0.42213723222752964, + "pass@64": 0.4665446501053006, + "pass@128": 0.5089820359281437 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 202404, + "answer_accuracy": 0.48327666564792177, + "avg_response_len": 358.6269841648839, + "resp_tokens_sum": 150198719, + "pass_at_k": { + "pass@1": 0.48327666564792177, + "pass@2": 0.5491766139686791, + "pass@4": 0.6044340943380715, + "pass@8": 0.6505155359408876, + "pass@16": 0.6888298655706238, + "pass@32": 0.7210480148449449, + "pass@64": 0.7492617331321013, + "pass@128": 0.773838630806846 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9474950396825397, + "pass@2": 0.9650906917885265, + "pass@4": 0.9757059212836491, + "pass@8": 0.9821219775598945, + "pass@16": 0.9864303866175479, + "pass@32": 0.9894757388321294, + "pass@64": 0.9917828226385678, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8642488532110092, + "pass@2": 0.9062210292807438, + "pass@4": 0.9305711077973047, + "pass@8": 0.9457780911926887, + "pass@16": 0.9560377547170604, + "pass@32": 0.9632593167850193, + "pass@64": 0.9692578704014565, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7041377314814815, + "pass@2": 0.7725702038981236, + "pass@4": 0.8198451316270648, + "pass@8": 0.8536786876304862, + "pass@16": 0.8791133247279139, + "pass@32": 0.8997977418138592, + "pass@64": 0.9182787748867802, + "pass@128": 0.9351851851851852 + }, + "14": { + "pass@1": 0.6734514094955489, + "pass@2": 0.7521295036098973, + "pass@4": 0.8051161497987822, + "pass@8": 0.8397737848596823, + "pass@16": 0.8640425599739608, + "pass@32": 0.8839021987844294, + "pass@64": 0.9026402228038988, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.5394724151234568, + "pass@2": 0.6320454517838044, + "pass@4": 0.6998636646113677, + "pass@8": 0.7534191486172014, + "pass@16": 0.7985213575068816, + "pass@32": 0.8361415781784669, + "pass@64": 0.8662591760380323, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.29194321533923306, + "pass@2": 0.3769282192878544, + "pass@4": 0.45713286143435594, + "pass@8": 0.5280299735269007, + "pass@16": 0.5870147719712617, + "pass@32": 0.6351865943510273, + "pass@64": 0.6781985568761224, + "pass@128": 0.7168141592920354 + }, + "17": { + "pass@1": 0.2419992469879518, + "pass@2": 0.31355688502039636, + "pass@4": 0.38367240164256594, + "pass@8": 0.44981450889515695, + "pass@16": 0.509154848544018, + "pass@32": 0.5591498560356163, + "pass@64": 0.6029538511472281, + "pass@128": 0.6445783132530121 + }, + "18": { + "pass@1": 0.24574092741935483, + "pass@2": 0.31357791465582935, + "pass@4": 0.37963863015106997, + "pass@8": 0.44168279089796914, + "pass@16": 0.4976640372324467, + "pass@32": 0.5469669934009332, + "pass@64": 0.5907081005343858, + "pass@128": 0.6258064516129033 + }, + "19": { + "pass@1": 0.1857533987915408, + "pass@2": 0.2546417441777481, + "pass@4": 0.3197634204334731, + "pass@8": 0.37426247715874117, + "pass@16": 0.4187446732966611, + "pass@32": 0.4566777531079154, + "pass@64": 0.48765619281340333, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.1602149024024024, + "pass@2": 0.2241135081932327, + "pass@4": 0.2897222278521489, + "pass@8": 0.35135852044599114, + "pass@16": 0.4049479489388728, + "pass@32": 0.4522311040905318, + "pass@64": 0.49616134090835173, + "pass@128": 0.5315315315315315 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7252191611842105, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7252191611842105, + "pass@2": 0.7601283218503743, + "pass@4": 0.7889876920697241, + "pass@8": 0.8128791845325699, + "pass@16": 0.8329568442979384, + "pass@32": 0.8502317417342605, + "pass@64": 0.8654224839691903, + "pass@128": 0.8786842105263157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.958484375, + "pass@2": 0.9682966289370083, + "pass@4": 0.9761080770528682, + "pass@8": 0.9826235127203949, + "pass@16": 0.9877163443569874, + "pass@32": 0.991429265701168, + "pass@64": 0.9938933039035253, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998671875, + "pass@2": 0.9999927411417323, + "pass@4": 0.9999999690663667, + "pass@8": 0.9999999999998846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997265625, + "pass@2": 0.9999621062992126, + "pass@4": 0.9999991390138733, + "pass@8": 0.9999999996569364, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999609375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.99578125, + "pass@2": 0.9979758858267715, + "pass@4": 0.9988914759092613, + "pass@8": 0.9990593780831115, + "pass@16": 0.9991249987361553, + "pass@32": 0.99925, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993640625, + "pass@2": 0.9954298720472442, + "pass@4": 0.9967109700037493, + "pass@8": 0.9977585535657889, + "pass@16": 0.9987166677518824, + "pass@32": 0.9994809308280161, + "pass@64": 0.9999263116699708, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9446015625, + "pass@2": 0.9606361958661419, + "pass@4": 0.9706209278215225, + "pass@8": 0.9770803766579949, + "pass@16": 0.9812540369298193, + "pass@32": 0.9842091557330336, + "pass@64": 0.9862806773451164, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8609765625, + "pass@2": 0.8986541584645682, + "pass@4": 0.9198446199850016, + "pass@8": 0.933813023195648, + "pass@16": 0.9433933345589797, + "pass@32": 0.9498562494029792, + "pass@64": 0.9550037480484834, + "pass@128": 0.96 + }, + "13": { + "pass@1": 0.7124296875, + "pass@2": 0.7845658218503946, + "pass@4": 0.831534217754031, + "pass@8": 0.862838242321, + "pass@16": 0.8858632772953222, + "pass@32": 0.9047596826854089, + "pass@64": 0.9210772857822727, + "pass@128": 0.935 + }, + "14": { + "pass@1": 0.6589375, + "pass@2": 0.7366493602362207, + "pass@4": 0.7909602792463437, + "pass@8": 0.8290085420254784, + "pass@16": 0.8571618068972121, + "pass@32": 0.8796693898656633, + "pass@64": 0.8985783168716919, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.5503046875, + "pass@2": 0.647400959645669, + "pass@4": 0.7197840649606297, + "pass@8": 0.7744435087875599, + "pass@16": 0.817383191828716, + "pass@32": 0.8523338002808312, + "pass@64": 0.8800684135832626, + "pass@128": 0.9 + }, + "16": { + "pass@1": 0.3044375, + "pass@2": 0.3845978100393697, + "pass@4": 0.4561933964191972, + "pass@8": 0.5182463992333959, + "pass@16": 0.571895842343343, + "pass@32": 0.617956005780948, + "pass@64": 0.6595758910037973, + "pass@128": 0.698 + }, + "17": { + "pass@1": 0.254734375, + "pass@2": 0.3320616387795277, + "pass@4": 0.40505249887514033, + "pass@8": 0.4699472794329776, + "pass@16": 0.5256080803835056, + "pass@32": 0.5731372708577546, + "pass@64": 0.6170483351733107, + "pass@128": 0.662 + }, + "18": { + "pass@1": 0.2149375, + "pass@2": 0.28408624507874014, + "pass@4": 0.3513472681852263, + "pass@8": 0.41372310769530835, + "pass@16": 0.4717159005894626, + "pass@32": 0.5261657332909438, + "pass@64": 0.5750761510177632, + "pass@128": 0.615 + }, + "19": { + "pass@1": 0.173078125, + "pass@2": 0.23374274114173227, + "pass@4": 0.29328206955380576, + "pass@8": 0.3477914039665271, + "pass@16": 0.3970700170662132, + "pass@32": 0.44177308812314764, + "pass@64": 0.4800497528756157, + "pass@128": 0.511 + }, + "20": { + "pass@1": 0.157265625, + "pass@2": 0.21838607283464567, + "pass@4": 0.2804371754780648, + "pass@8": 0.3383711787766657, + "pass@16": 0.38927654292301656, + "pass@32": 0.4343825204007902, + "pass@64": 0.4769490081397651, + "pass@128": 0.516 + } + }, + "per_op_accuracy": { + "10": 0.958484375, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9998671875, + "6": 0.9997265625, + "7": 0.9999609375, + "8": 0.99578125, + "9": 0.993640625, + "11": 0.9446015625, + "12": 0.8609765625, + "13": 0.7124296875, + "14": 0.6589375, + "15": 0.5503046875, + "16": 0.3044375, + "17": 0.254734375, + "18": 0.2149375, + "19": 0.173078125, + "20": 0.157265625 + }, + "per_op_avg_response_len": { + "10": 281.5871796875, + "2": 115.7971171875, + "3": 144.9716328125, + "4": 160.1594765625, + "5": 173.0815546875, + "6": 199.3073515625, + "7": 224.3540546875, + "8": 244.032203125, + "9": 259.9615859375, + "11": 299.2658984375, + "12": 307.45903125, + "13": 322.2324453125, + "14": 345.3673515625, + "15": 356.779328125, + "16": 346.7696484375, + "17": 351.8356640625, + "18": 354.310234375, + "19": 338.7199921875, + "20": 346.3409140625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.5871796875, + "2": 115.7971171875, + "3": 144.9716328125, + "4": 160.1594765625, + "5": 173.0815546875, + "6": 199.3073515625, + "7": 224.3540546875, + "8": 244.032203125, + "9": 259.9615859375, + "11": 299.2658984375, + "12": 307.45903125, + "13": 322.2324453125, + "14": 345.3673515625, + "15": 356.779328125, + "16": 346.7696484375, + "17": 351.8356640625, + "18": 354.310234375, + "19": 338.7199921875, + "20": 346.3409140625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 587392, + "answer_accuracy": 0.725418906101802, + "avg_response_len": 259.6781536515966, + "resp_tokens_sum": 210268672, + "pass_at_k": { + "pass@1": 0.725418906101802, + "pass@2": 0.7600856653954012, + "pass@4": 0.7884395743558797, + "pass@8": 0.8117030725400315, + "pass@16": 0.8313472866316567, + "pass@32": 0.8483059622741245, + "pass@64": 0.8631450176274401, + "pass@128": 0.8759089472020234 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9631959808259587, + "pass@2": 0.9731588100713074, + "pass@4": 0.9796233927617454, + "pass@8": 0.9842158548331358, + "pass@16": 0.9880032489114736, + "pass@32": 0.9909717471897269, + "pass@64": 0.9931521416822178, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994620134730539, + "pass@2": 0.9999068054835212, + "pass@4": 0.9999975148128939, + "pass@8": 0.9999999989732089, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956931089743589, + "pass@2": 0.9986860867151223, + "pass@4": 0.9998419906045398, + "pass@8": 0.9999969647115807, + "pass@16": 0.9999999989745152, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945642605633803, + "pass@2": 0.9965291255406454, + "pass@4": 0.9975614355423881, + "pass@8": 0.9981008476359892, + "pass@16": 0.9987574369960317, + "pass@32": 0.9995185120066794, + "pass@64": 0.999961060003192, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9376991855524079, + "pass@2": 0.9547899333050791, + "pass@4": 0.9666048129429994, + "pass@8": 0.9747402292502628, + "pass@16": 0.9798547128476756, + "pass@32": 0.9834549289935417, + "pass@64": 0.9858023195208817, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8657530014124294, + "pass@2": 0.9054356983184302, + "pass@4": 0.9276475596800396, + "pass@8": 0.9411308026374295, + "pass@16": 0.9485681266608572, + "pass@32": 0.9527450938166466, + "pass@64": 0.9565891063317338, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7410854231974922, + "pass@2": 0.807538629575692, + "pass@4": 0.8486559937257053, + "pass@8": 0.8765056311963663, + "pass@16": 0.8984341628770319, + "pass@32": 0.9170324322596793, + "pass@64": 0.9311435811665233, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6596816588785047, + "pass@2": 0.7336943023769223, + "pass@4": 0.7882888003485545, + "pass@8": 0.8288986917843586, + "pass@16": 0.8592432109531271, + "pass@32": 0.882945852339974, + "pass@64": 0.9017789483871335, + "pass@128": 0.9158878504672897 + }, + "15": { + "pass@1": 0.5488168352601156, + "pass@2": 0.6472950138819349, + "pass@4": 0.7241442177025557, + "pass@8": 0.7831827839828602, + "pass@16": 0.8300831095008099, + "pass@32": 0.867248365688248, + "pass@64": 0.8934960977796641, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.310710685483871, + "pass@2": 0.3919696151892304, + "pass@4": 0.45937909666533605, + "pass@8": 0.5143017852386241, + "pass@16": 0.5637792315815162, + "pass@32": 0.6079851621930701, + "pass@64": 0.6491849484380694, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.25593840317919075, + "pass@2": 0.33026311672204245, + "pass@4": 0.3988996206579669, + "pass@8": 0.4607312429836123, + "pass@16": 0.514943009962257, + "pass@32": 0.561849951338928, + "pass@64": 0.6068011159502685, + "pass@128": 0.653179190751445 + }, + "18": { + "pass@1": 0.19513392857142858, + "pass@2": 0.2628627671541054, + "pass@4": 0.3290206976806473, + "pass@8": 0.38876375862284673, + "pass@16": 0.4435721695894926, + "pass@32": 0.4966436629883193, + "pass@64": 0.5472989033830987, + "pass@128": 0.5914285714285714 + }, + "19": { + "pass@1": 0.1796875, + "pass@2": 0.23905499988177165, + "pass@4": 0.2972096981683596, + "pass@8": 0.3521358209569636, + "pass@16": 0.4022299045017779, + "pass@32": 0.44642726263310534, + "pass@64": 0.4824560579294367, + "pass@128": 0.5045045045045045 + }, + "20": { + "pass@1": 0.1664085960960961, + "pass@2": 0.22833537277434907, + "pass@4": 0.28835667529171466, + "pass@8": 0.3416753045571554, + "pass@16": 0.38789227690805683, + "pass@32": 0.42881599752146604, + "pass@64": 0.4681722777240641, + "pass@128": 0.5075075075075075 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 593783, + "answer_accuracy": 0.7231379091971941, + "avg_response_len": 268.6923787022603, + "resp_tokens_sum": 220628686, + "pass_at_k": { + "pass@1": 0.7231379091971941, + "pass@2": 0.758084885786875, + "pass@4": 0.7868573528659605, + "pass@8": 0.810708765477549, + "pass@16": 0.8309264952504994, + "pass@32": 0.8487348671005692, + "pass@64": 0.8645942957669521, + "pass@128": 0.8787217459080281 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954011694785276, + "pass@2": 0.9650013133423504, + "pass@4": 0.9737770710984742, + "pass@8": 0.9814918018455097, + "pass@16": 0.9874433942451809, + "pass@32": 0.9919418373382088, + "pass@64": 0.9950911414749884, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999773550724638, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997537607449857, + "pass@2": 0.9999806110822824, + "pass@4": 0.9999999113649476, + "pass@8": 0.9999999999996694, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999322615606936, + "pass@2": 0.9999996444176414, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9950704225352113, + "pass@2": 0.9965544249750472, + "pass@4": 0.9971638872605713, + "pass@8": 0.9973558977814235, + "pass@16": 0.9975352095202358, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992000374251497, + "pass@2": 0.9938727368098449, + "pass@4": 0.9951782018078579, + "pass@8": 0.9962647888400472, + "pass@16": 0.9976112410089141, + "pass@32": 0.9989591763936809, + "pass@64": 0.9998207645839522, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.949195218373494, + "pass@2": 0.962625847879708, + "pass@4": 0.9700664567719697, + "pass@8": 0.9747850987990536, + "pass@16": 0.9778305768375364, + "pass@32": 0.9800141809222661, + "pass@64": 0.981568883804507, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8523217084639498, + "pass@2": 0.883371926912349, + "pass@4": 0.9001900677383976, + "pass@8": 0.9134272201943227, + "pass@16": 0.9246892536945397, + "pass@32": 0.9329112526745617, + "pass@64": 0.9386328551278152, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.6943496148459384, + "pass@2": 0.7749250782990365, + "pass@4": 0.8268435101074553, + "pass@8": 0.8589385187033101, + "pass@16": 0.8807564765425908, + "pass@32": 0.8982965502713754, + "pass@64": 0.9146222978342695, + "pass@128": 0.9299719887955182 + }, + "14": { + "pass@1": 0.643937317251462, + "pass@2": 0.7241691416862364, + "pass@4": 0.7795188065855806, + "pass@8": 0.8185037908332939, + "pass@16": 0.8484280510819681, + "pass@32": 0.872423187936195, + "pass@64": 0.8915716940190292, + "pass@128": 0.9064327485380117 + }, + "15": { + "pass@1": 0.5625, + "pass@2": 0.6625883589835366, + "pass@4": 0.7347707221256433, + "pass@8": 0.7859226102348986, + "pass@16": 0.8225864124521411, + "pass@32": 0.8525939224935586, + "pass@64": 0.8795478506520482, + "pass@128": 0.9 + }, + "16": { + "pass@1": 0.3109642094017094, + "pass@2": 0.3854944814590484, + "pass@4": 0.4524724684521269, + "pass@8": 0.5122811532302066, + "pass@16": 0.5644623158256834, + "pass@32": 0.610120655316517, + "pass@64": 0.6507670832107987, + "pass@128": 0.6866096866096866 + }, + "17": { + "pass@1": 0.26657123447204967, + "pass@2": 0.3530736477233822, + "pass@4": 0.43370805211848495, + "pass@8": 0.5006082372902344, + "pass@16": 0.5540322646582369, + "pass@32": 0.5996876878591957, + "pass@64": 0.6425915232100543, + "pass@128": 0.6894409937888198 + }, + "18": { + "pass@1": 0.20723805147058824, + "pass@2": 0.2790444795044002, + "pass@4": 0.34853543720637853, + "pass@8": 0.41392390293806336, + "pass@16": 0.47702879320906416, + "pass@32": 0.5375902449727734, + "pass@64": 0.5894177166706436, + "pass@128": 0.6294117647058823 + }, + "19": { + "pass@1": 0.15404110863095238, + "pass@2": 0.20788990731627285, + "pass@4": 0.2633022258936383, + "pass@8": 0.317408618000936, + "pass@16": 0.37060408335097106, + "pass@32": 0.422477599367868, + "pass@64": 0.4701716838210624, + "pass@128": 0.5148809523809523 + }, + "20": { + "pass@1": 0.14520958083832336, + "pass@2": 0.20275627386958367, + "pass@4": 0.2632841339256246, + "pass@8": 0.32212848817550394, + "pass@16": 0.3750321787903876, + "pass@32": 0.42213723222752964, + "pass@64": 0.4665446501053006, + "pass@128": 0.5089820359281437 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 582558, + "answer_accuracy": 0.7271504034190766, + "avg_response_len": 288.53603685692605, + "resp_tokens_sum": 231161223, + "pass_at_k": { + "pass@1": 0.7271504034190766, + "pass@2": 0.7622658018123133, + "pass@4": 0.791725113167367, + "pass@8": 0.8162924011330738, + "pass@16": 0.8366645853006743, + "pass@32": 0.8537123187656835, + "pass@64": 0.8685731596993883, + "pass@128": 0.8814507109761943 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9580690298507463, + "pass@2": 0.9665831766365026, + "pass@4": 0.9748191693389073, + "pass@8": 0.9821134642696296, + "pass@16": 0.9876916312002068, + "pass@32": 0.9913934460644604, + "pass@64": 0.9934776589624096, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999790122742009, + "pass@4": 0.9999999090187256, + "pass@8": 0.9999999999996606, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999753548895899, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9966216216216216, + "pass@2": 0.9988258435600955, + "pass@4": 0.999842627216643, + "pass@8": 0.9999969410831621, + "pass@16": 0.9999999990282966, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943478697749196, + "pass@2": 0.9958473903081245, + "pass@4": 0.9973863054095731, + "pass@8": 0.9989720681107301, + "pass@16": 0.9998573090717485, + "pass@32": 0.999998373473169, + "pass@64": 0.9999999999929186, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9474950396825397, + "pass@2": 0.9650906917885265, + "pass@4": 0.9757059212836491, + "pass@8": 0.9821219775598945, + "pass@16": 0.9864303866175479, + "pass@32": 0.9894757388321294, + "pass@64": 0.9917828226385678, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8642488532110092, + "pass@2": 0.9062210292807438, + "pass@4": 0.9305711077973047, + "pass@8": 0.9457780911926887, + "pass@16": 0.9560377547170604, + "pass@32": 0.9632593167850193, + "pass@64": 0.9692578704014565, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7041377314814815, + "pass@2": 0.7725702038981236, + "pass@4": 0.8198451316270648, + "pass@8": 0.8536786876304862, + "pass@16": 0.8791133247279139, + "pass@32": 0.8997977418138592, + "pass@64": 0.9182787748867802, + "pass@128": 0.9351851851851852 + }, + "14": { + "pass@1": 0.6734514094955489, + "pass@2": 0.7521295036098973, + "pass@4": 0.8051161497987822, + "pass@8": 0.8397737848596823, + "pass@16": 0.8640425599739608, + "pass@32": 0.8839021987844294, + "pass@64": 0.9026402228038988, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.5394724151234568, + "pass@2": 0.6320454517838044, + "pass@4": 0.6998636646113677, + "pass@8": 0.7534191486172014, + "pass@16": 0.7985213575068816, + "pass@32": 0.8361415781784669, + "pass@64": 0.8662591760380323, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.29194321533923306, + "pass@2": 0.3769282192878544, + "pass@4": 0.45713286143435594, + "pass@8": 0.5280299735269007, + "pass@16": 0.5870147719712617, + "pass@32": 0.6351865943510273, + "pass@64": 0.6781985568761224, + "pass@128": 0.7168141592920354 + }, + "17": { + "pass@1": 0.2419992469879518, + "pass@2": 0.31355688502039636, + "pass@4": 0.38367240164256594, + "pass@8": 0.44981450889515695, + "pass@16": 0.509154848544018, + "pass@32": 0.5591498560356163, + "pass@64": 0.6029538511472281, + "pass@128": 0.6445783132530121 + }, + "18": { + "pass@1": 0.24574092741935483, + "pass@2": 0.31357791465582935, + "pass@4": 0.37963863015106997, + "pass@8": 0.44168279089796914, + "pass@16": 0.4976640372324467, + "pass@32": 0.5469669934009332, + "pass@64": 0.5907081005343858, + "pass@128": 0.6258064516129033 + }, + "19": { + "pass@1": 0.1857533987915408, + "pass@2": 0.2546417441777481, + "pass@4": 0.3197634204334731, + "pass@8": 0.37426247715874117, + "pass@16": 0.4187446732966611, + "pass@32": 0.4566777531079154, + "pass@64": 0.48765619281340333, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.1602149024024024, + "pass@2": 0.2241135081932327, + "pass@4": 0.2897222278521489, + "pass@8": 0.35135852044599114, + "pass@16": 0.4049479489388728, + "pass@32": 0.4522311040905318, + "pass@64": 0.49616134090835173, + "pass@128": 0.5315315315315315 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3ba7a5e352e8f9299a914d6a97052f17afcd5a9c --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cbc97f406c8a285c185aba290e7c4d85f10b815a11caecd80811c5449b3a985 +size 2595381297 diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/summary.csv b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..6efb7c53893069c66aef4ac7e4ca1e307d535d37 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op7-10,0.9941623263888889,0.4831703125,0.7252191611842105,0.0,0.0,0.0,200.36135069444444,336.90805078125,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/summary.json b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9872379366a558e60d451664193c7712a1779487 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op7-10", + "id_acc": 0.9941623263888889, + "ood_acc": 0.4831703125, + "total_acc": 0.7252191611842105, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.36135069444444, + "ood_avg_resp_len": 336.90805078125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 587392, + "answer_accuracy": 0.725418906101802, + "avg_response_len": 259.6781536515966, + "resp_tokens_sum": 210268672, + "pass_at_k": { + "pass@1": 0.725418906101802, + "pass@2": 0.7600856653954012, + "pass@4": 0.7884395743558797, + "pass@8": 0.8117030725400315, + "pass@16": 0.8313472866316567, + "pass@32": 0.8483059622741245, + "pass@64": 0.8631450176274401, + "pass@128": 0.8759089472020234 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9631959808259587, + "pass@2": 0.9731588100713074, + "pass@4": 0.9796233927617454, + "pass@8": 0.9842158548331358, + "pass@16": 0.9880032489114736, + "pass@32": 0.9909717471897269, + "pass@64": 0.9931521416822178, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994620134730539, + "pass@2": 0.9999068054835212, + "pass@4": 0.9999975148128939, + "pass@8": 0.9999999989732089, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956931089743589, + "pass@2": 0.9986860867151223, + "pass@4": 0.9998419906045398, + "pass@8": 0.9999969647115807, + "pass@16": 0.9999999989745152, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945642605633803, + "pass@2": 0.9965291255406454, + "pass@4": 0.9975614355423881, + "pass@8": 0.9981008476359892, + "pass@16": 0.9987574369960317, + "pass@32": 0.9995185120066794, + "pass@64": 0.999961060003192, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9376991855524079, + "pass@2": 0.9547899333050791, + "pass@4": 0.9666048129429994, + "pass@8": 0.9747402292502628, + "pass@16": 0.9798547128476756, + "pass@32": 0.9834549289935417, + "pass@64": 0.9858023195208817, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8657530014124294, + "pass@2": 0.9054356983184302, + "pass@4": 0.9276475596800396, + "pass@8": 0.9411308026374295, + "pass@16": 0.9485681266608572, + "pass@32": 0.9527450938166466, + "pass@64": 0.9565891063317338, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7410854231974922, + "pass@2": 0.807538629575692, + "pass@4": 0.8486559937257053, + "pass@8": 0.8765056311963663, + "pass@16": 0.8984341628770319, + "pass@32": 0.9170324322596793, + "pass@64": 0.9311435811665233, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6596816588785047, + "pass@2": 0.7336943023769223, + "pass@4": 0.7882888003485545, + "pass@8": 0.8288986917843586, + "pass@16": 0.8592432109531271, + "pass@32": 0.882945852339974, + "pass@64": 0.9017789483871335, + "pass@128": 0.9158878504672897 + }, + "15": { + "pass@1": 0.5488168352601156, + "pass@2": 0.6472950138819349, + "pass@4": 0.7241442177025557, + "pass@8": 0.7831827839828602, + "pass@16": 0.8300831095008099, + "pass@32": 0.867248365688248, + "pass@64": 0.8934960977796641, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.310710685483871, + "pass@2": 0.3919696151892304, + "pass@4": 0.45937909666533605, + "pass@8": 0.5143017852386241, + "pass@16": 0.5637792315815162, + "pass@32": 0.6079851621930701, + "pass@64": 0.6491849484380694, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.25593840317919075, + "pass@2": 0.33026311672204245, + "pass@4": 0.3988996206579669, + "pass@8": 0.4607312429836123, + "pass@16": 0.514943009962257, + "pass@32": 0.561849951338928, + "pass@64": 0.6068011159502685, + "pass@128": 0.653179190751445 + }, + "18": { + "pass@1": 0.19513392857142858, + "pass@2": 0.2628627671541054, + "pass@4": 0.3290206976806473, + "pass@8": 0.38876375862284673, + "pass@16": 0.4435721695894926, + "pass@32": 0.4966436629883193, + "pass@64": 0.5472989033830987, + "pass@128": 0.5914285714285714 + }, + "19": { + "pass@1": 0.1796875, + "pass@2": 0.23905499988177165, + "pass@4": 0.2972096981683596, + "pass@8": 0.3521358209569636, + "pass@16": 0.4022299045017779, + "pass@32": 0.44642726263310534, + "pass@64": 0.4824560579294367, + "pass@128": 0.5045045045045045 + }, + "20": { + "pass@1": 0.1664085960960961, + "pass@2": 0.22833537277434907, + "pass@4": 0.28835667529171466, + "pass@8": 0.3416753045571554, + "pass@16": 0.38789227690805683, + "pass@32": 0.42881599752146604, + "pass@64": 0.4681722777240641, + "pass@128": 0.5075075075075075 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 593783, + "answer_accuracy": 0.7231379091971941, + "avg_response_len": 268.6923787022603, + "resp_tokens_sum": 220628686, + "pass_at_k": { + "pass@1": 0.7231379091971941, + "pass@2": 0.758084885786875, + "pass@4": 0.7868573528659605, + "pass@8": 0.810708765477549, + "pass@16": 0.8309264952504994, + "pass@32": 0.8487348671005692, + "pass@64": 0.8645942957669521, + "pass@128": 0.8787217459080281 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954011694785276, + "pass@2": 0.9650013133423504, + "pass@4": 0.9737770710984742, + "pass@8": 0.9814918018455097, + "pass@16": 0.9874433942451809, + "pass@32": 0.9919418373382088, + "pass@64": 0.9950911414749884, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999773550724638, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997537607449857, + "pass@2": 0.9999806110822824, + "pass@4": 0.9999999113649476, + "pass@8": 0.9999999999996694, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999322615606936, + "pass@2": 0.9999996444176414, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9950704225352113, + "pass@2": 0.9965544249750472, + "pass@4": 0.9971638872605713, + "pass@8": 0.9973558977814235, + "pass@16": 0.9975352095202358, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992000374251497, + "pass@2": 0.9938727368098449, + "pass@4": 0.9951782018078579, + "pass@8": 0.9962647888400472, + "pass@16": 0.9976112410089141, + "pass@32": 0.9989591763936809, + "pass@64": 0.9998207645839522, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.949195218373494, + "pass@2": 0.962625847879708, + "pass@4": 0.9700664567719697, + "pass@8": 0.9747850987990536, + "pass@16": 0.9778305768375364, + "pass@32": 0.9800141809222661, + "pass@64": 0.981568883804507, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8523217084639498, + "pass@2": 0.883371926912349, + "pass@4": 0.9001900677383976, + "pass@8": 0.9134272201943227, + "pass@16": 0.9246892536945397, + "pass@32": 0.9329112526745617, + "pass@64": 0.9386328551278152, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.6943496148459384, + "pass@2": 0.7749250782990365, + "pass@4": 0.8268435101074553, + "pass@8": 0.8589385187033101, + "pass@16": 0.8807564765425908, + "pass@32": 0.8982965502713754, + "pass@64": 0.9146222978342695, + "pass@128": 0.9299719887955182 + }, + "14": { + "pass@1": 0.643937317251462, + "pass@2": 0.7241691416862364, + "pass@4": 0.7795188065855806, + "pass@8": 0.8185037908332939, + "pass@16": 0.8484280510819681, + "pass@32": 0.872423187936195, + "pass@64": 0.8915716940190292, + "pass@128": 0.9064327485380117 + }, + "15": { + "pass@1": 0.5625, + "pass@2": 0.6625883589835366, + "pass@4": 0.7347707221256433, + "pass@8": 0.7859226102348986, + "pass@16": 0.8225864124521411, + "pass@32": 0.8525939224935586, + "pass@64": 0.8795478506520482, + "pass@128": 0.9 + }, + "16": { + "pass@1": 0.3109642094017094, + "pass@2": 0.3854944814590484, + "pass@4": 0.4524724684521269, + "pass@8": 0.5122811532302066, + "pass@16": 0.5644623158256834, + "pass@32": 0.610120655316517, + "pass@64": 0.6507670832107987, + "pass@128": 0.6866096866096866 + }, + "17": { + "pass@1": 0.26657123447204967, + "pass@2": 0.3530736477233822, + "pass@4": 0.43370805211848495, + "pass@8": 0.5006082372902344, + "pass@16": 0.5540322646582369, + "pass@32": 0.5996876878591957, + "pass@64": 0.6425915232100543, + "pass@128": 0.6894409937888198 + }, + "18": { + "pass@1": 0.20723805147058824, + "pass@2": 0.2790444795044002, + "pass@4": 0.34853543720637853, + "pass@8": 0.41392390293806336, + "pass@16": 0.47702879320906416, + "pass@32": 0.5375902449727734, + "pass@64": 0.5894177166706436, + "pass@128": 0.6294117647058823 + }, + "19": { + "pass@1": 0.15404110863095238, + "pass@2": 0.20788990731627285, + "pass@4": 0.2633022258936383, + "pass@8": 0.317408618000936, + "pass@16": 0.37060408335097106, + "pass@32": 0.422477599367868, + "pass@64": 0.4701716838210624, + "pass@128": 0.5148809523809523 + }, + "20": { + "pass@1": 0.14520958083832336, + "pass@2": 0.20275627386958367, + "pass@4": 0.2632841339256246, + "pass@8": 0.32212848817550394, + "pass@16": 0.3750321787903876, + "pass@32": 0.42213723222752964, + "pass@64": 0.4665446501053006, + "pass@128": 0.5089820359281437 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 582558, + "answer_accuracy": 0.7271504034190766, + "avg_response_len": 288.53603685692605, + "resp_tokens_sum": 231161223, + "pass_at_k": { + "pass@1": 0.7271504034190766, + "pass@2": 0.7622658018123133, + "pass@4": 0.791725113167367, + "pass@8": 0.8162924011330738, + "pass@16": 0.8366645853006743, + "pass@32": 0.8537123187656835, + "pass@64": 0.8685731596993883, + "pass@128": 0.8814507109761943 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9580690298507463, + "pass@2": 0.9665831766365026, + "pass@4": 0.9748191693389073, + "pass@8": 0.9821134642696296, + "pass@16": 0.9876916312002068, + "pass@32": 0.9913934460644604, + "pass@64": 0.9934776589624096, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999790122742009, + "pass@4": 0.9999999090187256, + "pass@8": 0.9999999999996606, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999753548895899, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9966216216216216, + "pass@2": 0.9988258435600955, + "pass@4": 0.999842627216643, + "pass@8": 0.9999969410831621, + "pass@16": 0.9999999990282966, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9943478697749196, + "pass@2": 0.9958473903081245, + "pass@4": 0.9973863054095731, + "pass@8": 0.9989720681107301, + "pass@16": 0.9998573090717485, + "pass@32": 0.999998373473169, + "pass@64": 0.9999999999929186, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9474950396825397, + "pass@2": 0.9650906917885265, + "pass@4": 0.9757059212836491, + "pass@8": 0.9821219775598945, + "pass@16": 0.9864303866175479, + "pass@32": 0.9894757388321294, + "pass@64": 0.9917828226385678, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8642488532110092, + "pass@2": 0.9062210292807438, + "pass@4": 0.9305711077973047, + "pass@8": 0.9457780911926887, + "pass@16": 0.9560377547170604, + "pass@32": 0.9632593167850193, + "pass@64": 0.9692578704014565, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7041377314814815, + "pass@2": 0.7725702038981236, + "pass@4": 0.8198451316270648, + "pass@8": 0.8536786876304862, + "pass@16": 0.8791133247279139, + "pass@32": 0.8997977418138592, + "pass@64": 0.9182787748867802, + "pass@128": 0.9351851851851852 + }, + "14": { + "pass@1": 0.6734514094955489, + "pass@2": 0.7521295036098973, + "pass@4": 0.8051161497987822, + "pass@8": 0.8397737848596823, + "pass@16": 0.8640425599739608, + "pass@32": 0.8839021987844294, + "pass@64": 0.9026402228038988, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.5394724151234568, + "pass@2": 0.6320454517838044, + "pass@4": 0.6998636646113677, + "pass@8": 0.7534191486172014, + "pass@16": 0.7985213575068816, + "pass@32": 0.8361415781784669, + "pass@64": 0.8662591760380323, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.29194321533923306, + "pass@2": 0.3769282192878544, + "pass@4": 0.45713286143435594, + "pass@8": 0.5280299735269007, + "pass@16": 0.5870147719712617, + "pass@32": 0.6351865943510273, + "pass@64": 0.6781985568761224, + "pass@128": 0.7168141592920354 + }, + "17": { + "pass@1": 0.2419992469879518, + "pass@2": 0.31355688502039636, + "pass@4": 0.38367240164256594, + "pass@8": 0.44981450889515695, + "pass@16": 0.509154848544018, + "pass@32": 0.5591498560356163, + "pass@64": 0.6029538511472281, + "pass@128": 0.6445783132530121 + }, + "18": { + "pass@1": 0.24574092741935483, + "pass@2": 0.31357791465582935, + "pass@4": 0.37963863015106997, + "pass@8": 0.44168279089796914, + "pass@16": 0.4976640372324467, + "pass@32": 0.5469669934009332, + "pass@64": 0.5907081005343858, + "pass@128": 0.6258064516129033 + }, + "19": { + "pass@1": 0.1857533987915408, + "pass@2": 0.2546417441777481, + "pass@4": 0.3197634204334731, + "pass@8": 0.37426247715874117, + "pass@16": 0.4187446732966611, + "pass@32": 0.4566777531079154, + "pass@64": 0.48765619281340333, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.1602149024024024, + "pass@2": 0.2241135081932327, + "pass@4": 0.2897222278521489, + "pass@8": 0.35135852044599114, + "pass@16": 0.4049479489388728, + "pass@32": 0.4522311040905318, + "pass@64": 0.49616134090835173, + "pass@128": 0.5315315315315315 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e91baec6e4252f255e9058c21b8029e6cf16dc3c --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad32715f492dbe623554cdef060a01f6888caaeca744c1594fc11bbb2460833c +size 1789318589 diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f93ca3c4cbb4023ba834a36cb706353cb4e27483 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9937760416666667, + "avg_response_len": 199.80026041666667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9937760416666667, + "pass@2": 0.9955518099300088, + "pass@4": 0.996819273819939, + "pass@8": 0.9977502903983948, + "pass@16": 0.998421548032046, + "pass@32": 0.9989062220192162, + "pass@64": 0.9992576076995653, + "pass@128": 0.9994444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9579375, + "pass@2": 0.9683332923228342, + "pass@4": 0.9771047539370078, + "pass@8": 0.9840946719456297, + "pass@16": 0.9889645025655779, + "pass@32": 0.9923080022149289, + "pass@64": 0.9946954376025967, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998046875, + "pass@2": 0.999963090551181, + "pass@4": 0.9999988142107236, + "pass@8": 0.9999999992434965, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994375, + "pass@2": 0.9999210137795276, + "pass@4": 0.9999968714848144, + "pass@8": 0.9999999944732593, + "pass@16": 0.9999999999999968, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99940625, + "pass@2": 0.9997797736220473, + "pass@4": 0.999957270247469, + "pass@8": 0.9999984489972917, + "pass@16": 0.9999999988246276, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9943125, + "pass@2": 0.9968859498031498, + "pass@4": 0.9980599513498313, + "pass@8": 0.9985042871461746, + "pass@16": 0.998784194169822, + "pass@32": 0.9989637597380138, + "pass@64": 0.9989996944261581, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.993171875, + "pass@2": 0.9950875984251968, + "pass@4": 0.9962558149606298, + "pass@8": 0.9971552117797085, + "pass@16": 0.9980452367283879, + "pass@32": 0.9988842362200037, + "pass@64": 0.9996233372673325, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9579375, + "2": 1.0, + "3": 1.0, + "4": 0.9998046875, + "5": 0.9994375, + "6": 0.99940625, + "7": 0.9999140625, + "8": 0.9943125, + "9": 0.993171875 + }, + "per_op_avg_response_len": { + "10": 280.7613125, + "2": 115.56028125, + "3": 144.5886796875, + "4": 159.7724765625, + "5": 172.6244921875, + "6": 198.7315078125, + "7": 223.64196875, + "8": 243.3488828125, + "9": 259.1727421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7613125, + "2": 115.56028125, + "3": 144.5886796875, + "4": 159.7724765625, + "5": 172.6244921875, + "6": 198.7315078125, + "7": 223.64196875, + "8": 243.3488828125, + "9": 259.1727421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376817, + "answer_accuracy": 0.9942191193853428, + "avg_response_len": 190.34473414809187, + "resp_tokens_sum": 72142177, + "pass_at_k": { + "pass@1": 0.9942191193853428, + "pass@2": 0.9963355644374241, + "pass@4": 0.9975038111687408, + "pass@8": 0.9981769896688121, + "pass@16": 0.9986526147571787, + "pass@32": 0.9990022609170982, + "pass@64": 0.9992410590833083, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9619976032448377, + "pass@2": 0.9734375362924764, + "pass@4": 0.9813279509862154, + "pass@8": 0.9865472362157778, + "pass@16": 0.9902011494739077, + "pass@32": 0.9925198702388486, + "pass@64": 0.9937310418577298, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9993896484375, + "pass@2": 0.999884657972441, + "pass@4": 0.9999962944085115, + "pass@8": 0.9999999976359266, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9992311507936508, + "pass@2": 0.9998183820772404, + "pass@4": 0.9999906365870932, + "pass@8": 0.9999999824833706, + "pass@16": 0.9999999999999897, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996959206586826, + "pass@2": 0.9999712680937338, + "pass@4": 0.9999997993327181, + "pass@8": 0.9999999999973048, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997688609467456, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999813, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9937650240384616, + "pass@2": 0.9979112248889561, + "pass@4": 0.9995429453529848, + "pass@8": 0.9999526963573945, + "pass@16": 0.9999995305865249, + "pass@32": 0.9999999999899963, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952904929577465, + "pass@2": 0.9969411944105578, + "pass@4": 0.9974237313645655, + "pass@8": 0.9976825868840717, + "pass@16": 0.9981192931869252, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387830, + "answer_accuracy": 0.9927660140891219, + "avg_response_len": 197.81898140563564, + "resp_tokens_sum": 77279172, + "pass_at_k": { + "pass@1": 0.9927660140891219, + "pass@2": 0.9944949855393648, + "pass@4": 0.9958670114527789, + "pass@8": 0.9970192924862197, + "pass@16": 0.9979703268073893, + "pass@32": 0.998713234479722, + "pass@64": 0.9992014189425932, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9473015720858896, + "pass@2": 0.9586338673252502, + "pass@4": 0.9689450889190999, + "pass@8": 0.9781974552058535, + "pass@16": 0.9856057187041461, + "pass@32": 0.9914379875697714, + "pass@64": 0.995611394492944, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995244565217392, + "pass@2": 0.9999536403058312, + "pass@4": 0.9999995041380697, + "pass@8": 0.9999999999739078, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986568767908309, + "pass@2": 0.9993968283960923, + "pass@4": 0.9998777572216081, + "pass@8": 0.9999955558687446, + "pass@16": 0.9999999966321708, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959507042253521, + "pass@2": 0.9969051513807253, + "pass@4": 0.9971650308676204, + "pass@8": 0.9971830224753971, + "pass@16": 0.9971830985911607, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9915325598802395, + "pass@2": 0.9940396028808524, + "pass@4": 0.9956864305696818, + "pass@8": 0.9970421805475395, + "pass@16": 0.9984969290242817, + "pass@32": 0.999592837378342, + "pass@64": 0.9999802874493858, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380183, + "answer_accuracy": 0.9943688274188148, + "avg_response_len": 211.19787569049214, + "resp_tokens_sum": 80748551, + "pass_at_k": { + "pass@1": 0.9943688274188148, + "pass@2": 0.9958546994324489, + "pass@4": 0.9971136794626483, + "pass@8": 0.9980742104145491, + "pass@16": 0.9986535328343675, + "pass@32": 0.9990082062153687, + "pass@64": 0.9993314237487836, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9641791044776119, + "pass@2": 0.9726069455870255, + "pass@4": 0.9807715807912069, + "pass@8": 0.9873516073772327, + "pass@16": 0.9909816376011114, + "pass@32": 0.992940245540936, + "pass@64": 0.9947799994272432, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995404411764706, + "pass@2": 0.9999829927049559, + "pass@4": 0.9999999765654294, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930790165165165, + "pass@2": 0.9959048615544679, + "pass@4": 0.9976245238489334, + "pass@8": 0.9985557744861908, + "pass@16": 0.9993523742551476, + "pass@32": 0.9998911703937988, + "pass@64": 0.9999990823608353, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925140675241158, + "pass@2": 0.9940972573866369, + "pass@4": 0.9955341559997924, + "pass@8": 0.9966746145755141, + "pass@16": 0.9974756056363967, + "pass@32": 0.998195401826233, + "pass@64": 0.9992024710718075, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5772890625, + "avg_response_len": 354.19277734375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5772890625, + "pass@2": 0.6462853223425093, + "pass@4": 0.702933030961746, + "pass@8": 0.7485155239527209, + "pass@16": 0.7848493560165596, + "pass@32": 0.8136574385292977, + "pass@64": 0.8370120244345786, + "pass@128": 0.8572 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9584375, + "pass@2": 0.9691230314960633, + "pass@4": 0.9751204714098238, + "pass@8": 0.9790695353821208, + "pass@16": 0.9819457773535026, + "pass@32": 0.9843443936163968, + "pass@64": 0.9864906679151568, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.90275, + "pass@2": 0.9239185531496075, + "pass@4": 0.9366134455380583, + "pass@8": 0.9453314176381942, + "pass@16": 0.952677818879833, + "pass@32": 0.9594730735464847, + "pass@64": 0.9651640279084795, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.81790625, + "pass@2": 0.8652897391732293, + "pass@4": 0.89582160808024, + "pass@8": 0.9165588912635148, + "pass@16": 0.9313775227007297, + "pass@32": 0.9423672175958623, + "pass@64": 0.9513205139520854, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7877578125, + "pass@2": 0.8500775098425204, + "pass@4": 0.8889341581364837, + "pass@8": 0.9136667969110601, + "pass@16": 0.930362641684427, + "pass@32": 0.9420521540953944, + "pass@64": 0.9509347840612814, + "pass@128": 0.959 + }, + "15": { + "pass@1": 0.7064609375, + "pass@2": 0.789366018700788, + "pass@4": 0.8452894179790024, + "pass@8": 0.8844193385115139, + "pass@16": 0.9118198739513568, + "pass@32": 0.9303889879158427, + "pass@64": 0.9434209370319473, + "pass@128": 0.952 + }, + "16": { + "pass@1": 0.5035, + "pass@2": 0.6194383612204715, + "pass@4": 0.7093965236220481, + "pass@8": 0.7738245998517134, + "pass@16": 0.8198911747261802, + "pass@32": 0.8537872805893915, + "pass@64": 0.878237210318832, + "pass@128": 0.895 + }, + "17": { + "pass@1": 0.3887109375, + "pass@2": 0.4946059301181108, + "pass@4": 0.5852996705099356, + "pass@8": 0.6575081830243463, + "pass@16": 0.712805845481183, + "pass@32": 0.7528234367781991, + "pass@64": 0.7839639080321678, + "pass@128": 0.812 + }, + "18": { + "pass@1": 0.3141328125, + "pass@2": 0.40736687992126003, + "pass@4": 0.4943831467941501, + "pass@8": 0.5719055375445893, + "pass@16": 0.6396952074478515, + "pass@32": 0.6981638167087683, + "pass@64": 0.7460239636779524, + "pass@128": 0.786 + }, + "19": { + "pass@1": 0.208296875, + "pass@2": 0.2853204970472441, + "pass@4": 0.3635014985001874, + "pass@8": 0.4347487378404988, + "pass@16": 0.4952378298317788, + "pass@32": 0.5466486854939643, + "pass@64": 0.5931950927231626, + "pass@128": 0.637 + }, + "20": { + "pass@1": 0.1849375, + "pass@2": 0.2583467027559054, + "pass@4": 0.33497036904761857, + "pass@8": 0.40812220155969836, + "pass@16": 0.4726798681086095, + "pass@32": 0.5265253389526633, + "pass@64": 0.5713691387246295, + "pass@128": 0.613 + } + }, + "per_op_accuracy": { + "11": 0.9584375, + "12": 0.90275, + "13": 0.81790625, + "14": 0.7877578125, + "15": 0.7064609375, + "16": 0.5035, + "17": 0.3887109375, + "18": 0.3141328125, + "19": 0.208296875, + "20": 0.1849375 + }, + "per_op_avg_response_len": { + "11": 298.952125, + "12": 308.7133125, + "13": 326.8007421875, + "14": 353.2913984375, + "15": 370.0824375, + "16": 368.69259375, + "17": 378.7571328125, + "18": 386.35434375, + "19": 370.4365390625, + "20": 379.8471484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.952125, + "12": 308.7133125, + "13": 326.8007421875, + "14": 353.2913984375, + "15": 370.0824375, + "16": 368.69259375, + "17": 378.7571328125, + "18": 386.35434375, + "19": 370.4365390625, + "20": 379.8471484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 251526, + "answer_accuracy": 0.583966381872214, + "avg_response_len": 336.23645291604754, + "resp_tokens_sum": 144823765, + "pass_at_k": { + "pass@1": 0.583966381872214, + "pass@2": 0.6517798068935694, + "pass@4": 0.7064692072666278, + "pass@8": 0.7502279546380634, + "pass@16": 0.7853845989417383, + "pass@32": 0.8134451780632909, + "pass@64": 0.8359222783918243, + "pass@128": 0.8555720653789004 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9536782932011332, + "pass@2": 0.9652242588833622, + "pass@4": 0.9721828820618381, + "pass@8": 0.9764794430448108, + "pass@16": 0.9788281054613178, + "pass@32": 0.9806032397021716, + "pass@64": 0.9823113128676721, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.896826447740113, + "pass@2": 0.9179442479647675, + "pass@4": 0.9305790242851, + "pass@8": 0.9397665516265951, + "pass@16": 0.9472254013802787, + "pass@32": 0.9532869258033493, + "pass@64": 0.9573208647827872, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8412029780564263, + "pass@2": 0.8873328468639701, + "pass@4": 0.918331002511833, + "pass@8": 0.9388124431693992, + "pass@16": 0.951719498399668, + "pass@32": 0.9604790149561151, + "pass@64": 0.9673495268907015, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8067805685358256, + "pass@2": 0.8619935395049919, + "pass@4": 0.8958573317938054, + "pass@8": 0.9184229052436736, + "pass@16": 0.9341940408306993, + "pass@32": 0.9454088534696391, + "pass@64": 0.9546836397889742, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7217982297687862, + "pass@2": 0.8086309083564697, + "pass@4": 0.8671244056017556, + "pass@8": 0.9076261577759502, + "pass@16": 0.9361793648177896, + "pass@32": 0.9553006962363504, + "pass@64": 0.9674216402888633, + "pass@128": 0.976878612716763 + }, + "16": { + "pass@1": 0.5188508064516129, + "pass@2": 0.6318909544069088, + "pass@4": 0.7154870514169593, + "pass@8": 0.7761161641465719, + "pass@16": 0.8212375445719823, + "pass@32": 0.8539394390663827, + "pass@64": 0.8752827466227667, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.405301661849711, + "pass@2": 0.513899358813891, + "pass@4": 0.6023861773419938, + "pass@8": 0.6674118975416066, + "pass@16": 0.7150319798410362, + "pass@32": 0.7483390641538125, + "pass@64": 0.7720001182291946, + "pass@128": 0.7947976878612717 + }, + "18": { + "pass@1": 0.28941964285714283, + "pass@2": 0.3808032199100109, + "pass@4": 0.465654920456371, + "pass@8": 0.5423820846954517, + "pass@16": 0.6123445134417186, + "pass@32": 0.6757494775720401, + "pass@64": 0.728451061773362, + "pass@128": 0.7685714285714286 + }, + "19": { + "pass@1": 0.20218656156156156, + "pass@2": 0.27512687392116536, + "pass@4": 0.34533306365983546, + "pass@8": 0.40891840561059223, + "pass@16": 0.4650283703236653, + "pass@32": 0.5150359397845146, + "pass@64": 0.5610112974633917, + "pass@128": 0.6036036036036037 + }, + "20": { + "pass@1": 0.1926614114114114, + "pass@2": 0.268538962190537, + "pass@4": 0.34810443598829427, + "pass@8": 0.4242786966820873, + "pass@16": 0.49075814995394723, + "pass@32": 0.5452179963812089, + "pass@64": 0.5923147933525963, + "pass@128": 0.6396396396396397 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 250768, + "answer_accuracy": 0.5825527802557241, + "avg_response_len": 349.6598879348796, + "resp_tokens_sum": 150515994, + "pass_at_k": { + "pass@1": 0.5825527802557241, + "pass@2": 0.6520572709967919, + "pass@4": 0.7085842892241074, + "pass@8": 0.7536817863047192, + "pass@16": 0.7896893855639213, + "pass@32": 0.8185435320264949, + "pass@64": 0.8418034312349038, + "pass@128": 0.8614332441272673 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.959972703313253, + "pass@2": 0.9685469239161373, + "pass@4": 0.9733140828217254, + "pass@8": 0.9767761134345652, + "pass@16": 0.9800878380863752, + "pass@32": 0.9835276964020698, + "pass@64": 0.9863941977444657, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9062010188087775, + "pass@2": 0.9236768907511169, + "pass@4": 0.9325341756849359, + "pass@8": 0.9386648112986568, + "pass@16": 0.9453690133693744, + "pass@32": 0.9528166752988761, + "pass@64": 0.96035720610297, + "pass@128": 0.9686520376175548 + }, + "13": { + "pass@1": 0.8154105392156863, + "pass@2": 0.8603254786166442, + "pass@4": 0.8889493970816668, + "pass@8": 0.9097444640506971, + "pass@16": 0.9264317584317059, + "pass@32": 0.9395132059440757, + "pass@64": 0.9505331010668854, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.7924433479532164, + "pass@2": 0.8606995671593685, + "pass@4": 0.9008573744619639, + "pass@8": 0.9240949005993414, + "pass@16": 0.9385674595627927, + "pass@32": 0.9479368741078822, + "pass@64": 0.9556970023204434, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.7134232954545454, + "pass@2": 0.7998765956812215, + "pass@4": 0.8590623126654622, + "pass@8": 0.8993640314330371, + "pass@16": 0.9265664600231758, + "pass@32": 0.944361073070663, + "pass@64": 0.9567259670636806, + "pass@128": 0.9636363636363636 + }, + "16": { + "pass@1": 0.5023593304843305, + "pass@2": 0.6129323978733429, + "pass@4": 0.7012541896472335, + "pass@8": 0.767042798214594, + "pass@16": 0.813623451502168, + "pass@32": 0.8470492421822465, + "pass@64": 0.8704872354663128, + "pass@128": 0.8831908831908832 + }, + "17": { + "pass@1": 0.40945263975155277, + "pass@2": 0.5173417096640096, + "pass@4": 0.6044674413757283, + "pass@8": 0.671494534674276, + "pass@16": 0.7233635296161987, + "pass@32": 0.7620572120609392, + "pass@64": 0.7921207132131408, + "pass@128": 0.8167701863354038 + }, + "18": { + "pass@1": 0.33120404411764703, + "pass@2": 0.4317359744094487, + "pass@4": 0.5253682403670127, + "pass@8": 0.607578615439266, + "pass@16": 0.6789513923043637, + "pass@32": 0.7411219173875988, + "pass@64": 0.7907025142353047, + "pass@128": 0.8294117647058824 + }, + "19": { + "pass@1": 0.21430896577380953, + "pass@2": 0.28969486724315685, + "pass@4": 0.36861701020631393, + "pass@8": 0.44152619961345574, + "pass@16": 0.5023052262297876, + "pass@32": 0.5512420329630123, + "pass@64": 0.5924569517750432, + "pass@128": 0.6309523809523809 + }, + "20": { + "pass@1": 0.18263473053892215, + "pass@2": 0.25554599462492333, + "pass@4": 0.3297973451297629, + "pass@8": 0.397951940647655, + "pass@16": 0.45852535327345006, + "pass@32": 0.5124159362936721, + "pass@64": 0.559076574550227, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 236636, + "answer_accuracy": 0.5650118429095354, + "avg_response_len": 377.3184310055012, + "resp_tokens_sum": 158026996, + "pass_at_k": { + "pass@1": 0.5650118429095354, + "pass@2": 0.6347021915850096, + "pass@4": 0.6934879164134142, + "pass@8": 0.7414444758030065, + "pass@16": 0.7793242607197735, + "pass@32": 0.8088537478315614, + "pass@64": 0.8332080801693723, + "pass@128": 0.8545232273838631 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9621527777777777, + "pass@2": 0.9740993313335832, + "pass@4": 0.980316312841847, + "pass@8": 0.9843892772286572, + "pass@16": 0.9873977583523199, + "pass@32": 0.9893976342731529, + "pass@64": 0.9912758756244631, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9057960626911316, + "pass@2": 0.9306218997567968, + "pass@4": 0.9471255929591371, + "pass@8": 0.957859276936109, + "pass@16": 0.9657104328023974, + "pass@32": 0.972663554714854, + "pass@64": 0.9783440153777522, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7977189429012346, + "pass@2": 0.8490566825848157, + "pass@4": 0.8812317392617589, + "pass@8": 0.9021572476122781, + "pass@16": 0.9167989658985097, + "pass@32": 0.9276796213050199, + "pass@64": 0.9364064746699794, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7648831602373887, + "pass@2": 0.8279475717890604, + "pass@4": 0.8702395625064661, + "pass@8": 0.8985536745485637, + "pass@16": 0.9183865976477663, + "pass@32": 0.9328827928390062, + "pass@64": 0.9425310412327254, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6829909336419753, + "pass@2": 0.7580878019587827, + "pass@4": 0.8079438594481239, + "pass@8": 0.844415301074487, + "pass@16": 0.8707866725825771, + "pass@32": 0.889554916372676, + "pass@64": 0.9042391369166215, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4906434365781711, + "pass@2": 0.6147872970524704, + "pass@4": 0.712257572615369, + "pass@8": 0.7787509344924866, + "pass@16": 0.825149570594711, + "pass@32": 0.8606246916603065, + "pass@64": 0.8889632425283108, + "pass@128": 0.911504424778761 + }, + "17": { + "pass@1": 0.35130365210843373, + "pass@2": 0.45244795619485817, + "pass@4": 0.5489022199597038, + "pass@8": 0.6336217659934749, + "pass@16": 0.7002461563848493, + "pass@32": 0.7485412599318023, + "pass@64": 0.7885210767175737, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.32331149193548386, + "pass@2": 0.41063071501142995, + "pass@4": 0.4928345900310848, + "pass@8": 0.5661131569413892, + "pass@16": 0.6275198527734406, + "pass@32": 0.6763549924089363, + "pass@64": 0.7168620555395872, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.20834120090634442, + "pass@2": 0.2911352647667531, + "pass@4": 0.37658692710616615, + "pass@8": 0.45385530423580156, + "pass@16": 0.5184556691866746, + "pass@32": 0.5737897113901778, + "pass@64": 0.6263226431161294, + "pass@128": 0.676737160120846 + }, + "20": { + "pass@1": 0.17952327327327328, + "pass@2": 0.2509635619871839, + "pass@4": 0.3270248606311598, + "pass@8": 0.40216650867341297, + "pass@16": 0.46879860714898774, + "pass@32": 0.5219844547616636, + "pass@64": 0.5627529628781948, + "pass@128": 0.5945945945945946 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7745723684210526, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7745723684210526, + "pass@2": 0.8117273427786826, + "pass@4": 0.8421423038945633, + "pass@8": 0.8665740975322662, + "pass@16": 0.8860151311817842, + "pass@32": 0.9014068622876938, + "pass@64": 0.9138651954548331, + "pass@128": 0.9245789473684211 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9579375, + "pass@2": 0.9683332923228342, + "pass@4": 0.9771047539370078, + "pass@8": 0.9840946719456297, + "pass@16": 0.9889645025655779, + "pass@32": 0.9923080022149289, + "pass@64": 0.9946954376025967, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998046875, + "pass@2": 0.999963090551181, + "pass@4": 0.9999988142107236, + "pass@8": 0.9999999992434965, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994375, + "pass@2": 0.9999210137795276, + "pass@4": 0.9999968714848144, + "pass@8": 0.9999999944732593, + "pass@16": 0.9999999999999968, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99940625, + "pass@2": 0.9997797736220473, + "pass@4": 0.999957270247469, + "pass@8": 0.9999984489972917, + "pass@16": 0.9999999988246276, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9943125, + "pass@2": 0.9968859498031498, + "pass@4": 0.9980599513498313, + "pass@8": 0.9985042871461746, + "pass@16": 0.998784194169822, + "pass@32": 0.9989637597380138, + "pass@64": 0.9989996944261581, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.993171875, + "pass@2": 0.9950875984251968, + "pass@4": 0.9962558149606298, + "pass@8": 0.9971552117797085, + "pass@16": 0.9980452367283879, + "pass@32": 0.9988842362200037, + "pass@64": 0.9996233372673325, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9584375, + "pass@2": 0.9691230314960633, + "pass@4": 0.9751204714098238, + "pass@8": 0.9790695353821208, + "pass@16": 0.9819457773535026, + "pass@32": 0.9843443936163968, + "pass@64": 0.9864906679151568, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.90275, + "pass@2": 0.9239185531496075, + "pass@4": 0.9366134455380583, + "pass@8": 0.9453314176381942, + "pass@16": 0.952677818879833, + "pass@32": 0.9594730735464847, + "pass@64": 0.9651640279084795, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.81790625, + "pass@2": 0.8652897391732293, + "pass@4": 0.89582160808024, + "pass@8": 0.9165588912635148, + "pass@16": 0.9313775227007297, + "pass@32": 0.9423672175958623, + "pass@64": 0.9513205139520854, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7877578125, + "pass@2": 0.8500775098425204, + "pass@4": 0.8889341581364837, + "pass@8": 0.9136667969110601, + "pass@16": 0.930362641684427, + "pass@32": 0.9420521540953944, + "pass@64": 0.9509347840612814, + "pass@128": 0.959 + }, + "15": { + "pass@1": 0.7064609375, + "pass@2": 0.789366018700788, + "pass@4": 0.8452894179790024, + "pass@8": 0.8844193385115139, + "pass@16": 0.9118198739513568, + "pass@32": 0.9303889879158427, + "pass@64": 0.9434209370319473, + "pass@128": 0.952 + }, + "16": { + "pass@1": 0.5035, + "pass@2": 0.6194383612204715, + "pass@4": 0.7093965236220481, + "pass@8": 0.7738245998517134, + "pass@16": 0.8198911747261802, + "pass@32": 0.8537872805893915, + "pass@64": 0.878237210318832, + "pass@128": 0.895 + }, + "17": { + "pass@1": 0.3887109375, + "pass@2": 0.4946059301181108, + "pass@4": 0.5852996705099356, + "pass@8": 0.6575081830243463, + "pass@16": 0.712805845481183, + "pass@32": 0.7528234367781991, + "pass@64": 0.7839639080321678, + "pass@128": 0.812 + }, + "18": { + "pass@1": 0.3141328125, + "pass@2": 0.40736687992126003, + "pass@4": 0.4943831467941501, + "pass@8": 0.5719055375445893, + "pass@16": 0.6396952074478515, + "pass@32": 0.6981638167087683, + "pass@64": 0.7460239636779524, + "pass@128": 0.786 + }, + "19": { + "pass@1": 0.208296875, + "pass@2": 0.2853204970472441, + "pass@4": 0.3635014985001874, + "pass@8": 0.4347487378404988, + "pass@16": 0.4952378298317788, + "pass@32": 0.5466486854939643, + "pass@64": 0.5931950927231626, + "pass@128": 0.637 + }, + "20": { + "pass@1": 0.1849375, + "pass@2": 0.2583467027559054, + "pass@4": 0.33497036904761857, + "pass@8": 0.40812220155969836, + "pass@16": 0.4726798681086095, + "pass@32": 0.5265253389526633, + "pass@64": 0.5713691387246295, + "pass@128": 0.613 + } + }, + "per_op_accuracy": { + "10": 0.9579375, + "2": 1.0, + "3": 1.0, + "4": 0.9998046875, + "5": 0.9994375, + "6": 0.99940625, + "7": 0.9999140625, + "8": 0.9943125, + "9": 0.993171875, + "11": 0.9584375, + "12": 0.90275, + "13": 0.81790625, + "14": 0.7877578125, + "15": 0.7064609375, + "16": 0.5035, + "17": 0.3887109375, + "18": 0.3141328125, + "19": 0.208296875, + "20": 0.1849375 + }, + "per_op_avg_response_len": { + "10": 280.7613125, + "2": 115.56028125, + "3": 144.5886796875, + "4": 159.7724765625, + "5": 172.6244921875, + "6": 198.7315078125, + "7": 223.64196875, + "8": 243.3488828125, + "9": 259.1727421875, + "11": 298.952125, + "12": 308.7133125, + "13": 326.8007421875, + "14": 353.2913984375, + "15": 370.0824375, + "16": 368.69259375, + "17": 378.7571328125, + "18": 386.35434375, + "19": 370.4365390625, + "20": 379.8471484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7613125, + "2": 115.56028125, + "3": 144.5886796875, + "4": 159.7724765625, + "5": 172.6244921875, + "6": 198.7315078125, + "7": 223.64196875, + "8": 243.3488828125, + "9": 259.1727421875, + "11": 298.952125, + "12": 308.7133125, + "13": 326.8007421875, + "14": 353.2913984375, + "15": 370.0824375, + "16": 368.69259375, + "17": 378.7571328125, + "18": 386.35434375, + "19": 370.4365390625, + "20": 379.8471484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 628343, + "answer_accuracy": 0.7759926790230793, + "avg_response_len": 267.9491656457477, + "resp_tokens_sum": 216965942, + "pass_at_k": { + "pass@1": 0.7759926790230793, + "pass@2": 0.8130554309984241, + "pass@4": 0.8426932765290545, + "pass@8": 0.8662850353724942, + "pass@16": 0.885208594330536, + "pass@32": 0.9002985644575572, + "pass@64": 0.9123666207294034, + "pass@128": 0.92285804615871 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9619976032448377, + "pass@2": 0.9734375362924764, + "pass@4": 0.9813279509862154, + "pass@8": 0.9865472362157778, + "pass@16": 0.9902011494739077, + "pass@32": 0.9925198702388486, + "pass@64": 0.9937310418577298, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9993896484375, + "pass@2": 0.999884657972441, + "pass@4": 0.9999962944085115, + "pass@8": 0.9999999976359266, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9992311507936508, + "pass@2": 0.9998183820772404, + "pass@4": 0.9999906365870932, + "pass@8": 0.9999999824833706, + "pass@16": 0.9999999999999897, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996959206586826, + "pass@2": 0.9999712680937338, + "pass@4": 0.9999997993327181, + "pass@8": 0.9999999999973048, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997688609467456, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999813, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9937650240384616, + "pass@2": 0.9979112248889561, + "pass@4": 0.9995429453529848, + "pass@8": 0.9999526963573945, + "pass@16": 0.9999995305865249, + "pass@32": 0.9999999999899963, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952904929577465, + "pass@2": 0.9969411944105578, + "pass@4": 0.9974237313645655, + "pass@8": 0.9976825868840717, + "pass@16": 0.9981192931869252, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9536782932011332, + "pass@2": 0.9652242588833622, + "pass@4": 0.9721828820618381, + "pass@8": 0.9764794430448108, + "pass@16": 0.9788281054613178, + "pass@32": 0.9806032397021716, + "pass@64": 0.9823113128676721, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.896826447740113, + "pass@2": 0.9179442479647675, + "pass@4": 0.9305790242851, + "pass@8": 0.9397665516265951, + "pass@16": 0.9472254013802787, + "pass@32": 0.9532869258033493, + "pass@64": 0.9573208647827872, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8412029780564263, + "pass@2": 0.8873328468639701, + "pass@4": 0.918331002511833, + "pass@8": 0.9388124431693992, + "pass@16": 0.951719498399668, + "pass@32": 0.9604790149561151, + "pass@64": 0.9673495268907015, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8067805685358256, + "pass@2": 0.8619935395049919, + "pass@4": 0.8958573317938054, + "pass@8": 0.9184229052436736, + "pass@16": 0.9341940408306993, + "pass@32": 0.9454088534696391, + "pass@64": 0.9546836397889742, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7217982297687862, + "pass@2": 0.8086309083564697, + "pass@4": 0.8671244056017556, + "pass@8": 0.9076261577759502, + "pass@16": 0.9361793648177896, + "pass@32": 0.9553006962363504, + "pass@64": 0.9674216402888633, + "pass@128": 0.976878612716763 + }, + "16": { + "pass@1": 0.5188508064516129, + "pass@2": 0.6318909544069088, + "pass@4": 0.7154870514169593, + "pass@8": 0.7761161641465719, + "pass@16": 0.8212375445719823, + "pass@32": 0.8539394390663827, + "pass@64": 0.8752827466227667, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.405301661849711, + "pass@2": 0.513899358813891, + "pass@4": 0.6023861773419938, + "pass@8": 0.6674118975416066, + "pass@16": 0.7150319798410362, + "pass@32": 0.7483390641538125, + "pass@64": 0.7720001182291946, + "pass@128": 0.7947976878612717 + }, + "18": { + "pass@1": 0.28941964285714283, + "pass@2": 0.3808032199100109, + "pass@4": 0.465654920456371, + "pass@8": 0.5423820846954517, + "pass@16": 0.6123445134417186, + "pass@32": 0.6757494775720401, + "pass@64": 0.728451061773362, + "pass@128": 0.7685714285714286 + }, + "19": { + "pass@1": 0.20218656156156156, + "pass@2": 0.27512687392116536, + "pass@4": 0.34533306365983546, + "pass@8": 0.40891840561059223, + "pass@16": 0.4650283703236653, + "pass@32": 0.5150359397845146, + "pass@64": 0.5610112974633917, + "pass@128": 0.6036036036036037 + }, + "20": { + "pass@1": 0.1926614114114114, + "pass@2": 0.268538962190537, + "pass@4": 0.34810443598829427, + "pass@8": 0.4242786966820873, + "pass@16": 0.49075814995394723, + "pass@32": 0.5452179963812089, + "pass@64": 0.5923147933525963, + "pass@128": 0.6396396396396397 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 638598, + "answer_accuracy": 0.7777158028059236, + "avg_response_len": 277.42006771239284, + "resp_tokens_sum": 227795166, + "pass_at_k": { + "pass@1": 0.7777158028059236, + "pass@2": 0.8149754167152435, + "pass@4": 0.8452618992384288, + "pass@8": 0.869452023072591, + "pass@16": 0.8887811131828001, + "pass@32": 0.9042610584313685, + "pass@64": 0.916687088052349, + "pass@128": 0.9270459859703819 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9473015720858896, + "pass@2": 0.9586338673252502, + "pass@4": 0.9689450889190999, + "pass@8": 0.9781974552058535, + "pass@16": 0.9856057187041461, + "pass@32": 0.9914379875697714, + "pass@64": 0.995611394492944, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995244565217392, + "pass@2": 0.9999536403058312, + "pass@4": 0.9999995041380697, + "pass@8": 0.9999999999739078, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986568767908309, + "pass@2": 0.9993968283960923, + "pass@4": 0.9998777572216081, + "pass@8": 0.9999955558687446, + "pass@16": 0.9999999966321708, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959507042253521, + "pass@2": 0.9969051513807253, + "pass@4": 0.9971650308676204, + "pass@8": 0.9971830224753971, + "pass@16": 0.9971830985911607, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9915325598802395, + "pass@2": 0.9940396028808524, + "pass@4": 0.9956864305696818, + "pass@8": 0.9970421805475395, + "pass@16": 0.9984969290242817, + "pass@32": 0.999592837378342, + "pass@64": 0.9999802874493858, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.959972703313253, + "pass@2": 0.9685469239161373, + "pass@4": 0.9733140828217254, + "pass@8": 0.9767761134345652, + "pass@16": 0.9800878380863752, + "pass@32": 0.9835276964020698, + "pass@64": 0.9863941977444657, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9062010188087775, + "pass@2": 0.9236768907511169, + "pass@4": 0.9325341756849359, + "pass@8": 0.9386648112986568, + "pass@16": 0.9453690133693744, + "pass@32": 0.9528166752988761, + "pass@64": 0.96035720610297, + "pass@128": 0.9686520376175548 + }, + "13": { + "pass@1": 0.8154105392156863, + "pass@2": 0.8603254786166442, + "pass@4": 0.8889493970816668, + "pass@8": 0.9097444640506971, + "pass@16": 0.9264317584317059, + "pass@32": 0.9395132059440757, + "pass@64": 0.9505331010668854, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.7924433479532164, + "pass@2": 0.8606995671593685, + "pass@4": 0.9008573744619639, + "pass@8": 0.9240949005993414, + "pass@16": 0.9385674595627927, + "pass@32": 0.9479368741078822, + "pass@64": 0.9556970023204434, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.7134232954545454, + "pass@2": 0.7998765956812215, + "pass@4": 0.8590623126654622, + "pass@8": 0.8993640314330371, + "pass@16": 0.9265664600231758, + "pass@32": 0.944361073070663, + "pass@64": 0.9567259670636806, + "pass@128": 0.9636363636363636 + }, + "16": { + "pass@1": 0.5023593304843305, + "pass@2": 0.6129323978733429, + "pass@4": 0.7012541896472335, + "pass@8": 0.767042798214594, + "pass@16": 0.813623451502168, + "pass@32": 0.8470492421822465, + "pass@64": 0.8704872354663128, + "pass@128": 0.8831908831908832 + }, + "17": { + "pass@1": 0.40945263975155277, + "pass@2": 0.5173417096640096, + "pass@4": 0.6044674413757283, + "pass@8": 0.671494534674276, + "pass@16": 0.7233635296161987, + "pass@32": 0.7620572120609392, + "pass@64": 0.7921207132131408, + "pass@128": 0.8167701863354038 + }, + "18": { + "pass@1": 0.33120404411764703, + "pass@2": 0.4317359744094487, + "pass@4": 0.5253682403670127, + "pass@8": 0.607578615439266, + "pass@16": 0.6789513923043637, + "pass@32": 0.7411219173875988, + "pass@64": 0.7907025142353047, + "pass@128": 0.8294117647058824 + }, + "19": { + "pass@1": 0.21430896577380953, + "pass@2": 0.28969486724315685, + "pass@4": 0.36861701020631393, + "pass@8": 0.44152619961345574, + "pass@16": 0.5023052262297876, + "pass@32": 0.5512420329630123, + "pass@64": 0.5924569517750432, + "pass@128": 0.6309523809523809 + }, + "20": { + "pass@1": 0.18263473053892215, + "pass@2": 0.25554599462492333, + "pass@4": 0.3297973451297629, + "pass@8": 0.397951940647655, + "pass@16": 0.45852535327345006, + "pass@32": 0.5124159362936721, + "pass@64": 0.559076574550227, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 616819, + "answer_accuracy": 0.769915072295894, + "avg_response_len": 298.0402557816744, + "resp_tokens_sum": 238775547, + "pass_at_k": { + "pass@1": 0.769915072295894, + "pass@2": 0.8070560086388907, + "pass@4": 0.8383880848473545, + "pass@8": 0.863916598711563, + "pass@16": 0.8839953800369682, + "pass@32": 0.8996016895462834, + "pass@64": 0.9124875860443917, + "pass@128": 0.9237897427704106 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9641791044776119, + "pass@2": 0.9726069455870255, + "pass@4": 0.9807715807912069, + "pass@8": 0.9873516073772327, + "pass@16": 0.9909816376011114, + "pass@32": 0.992940245540936, + "pass@64": 0.9947799994272432, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995404411764706, + "pass@2": 0.9999829927049559, + "pass@4": 0.9999999765654294, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930790165165165, + "pass@2": 0.9959048615544679, + "pass@4": 0.9976245238489334, + "pass@8": 0.9985557744861908, + "pass@16": 0.9993523742551476, + "pass@32": 0.9998911703937988, + "pass@64": 0.9999990823608353, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925140675241158, + "pass@2": 0.9940972573866369, + "pass@4": 0.9955341559997924, + "pass@8": 0.9966746145755141, + "pass@16": 0.9974756056363967, + "pass@32": 0.998195401826233, + "pass@64": 0.9992024710718075, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9621527777777777, + "pass@2": 0.9740993313335832, + "pass@4": 0.980316312841847, + "pass@8": 0.9843892772286572, + "pass@16": 0.9873977583523199, + "pass@32": 0.9893976342731529, + "pass@64": 0.9912758756244631, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9057960626911316, + "pass@2": 0.9306218997567968, + "pass@4": 0.9471255929591371, + "pass@8": 0.957859276936109, + "pass@16": 0.9657104328023974, + "pass@32": 0.972663554714854, + "pass@64": 0.9783440153777522, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7977189429012346, + "pass@2": 0.8490566825848157, + "pass@4": 0.8812317392617589, + "pass@8": 0.9021572476122781, + "pass@16": 0.9167989658985097, + "pass@32": 0.9276796213050199, + "pass@64": 0.9364064746699794, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7648831602373887, + "pass@2": 0.8279475717890604, + "pass@4": 0.8702395625064661, + "pass@8": 0.8985536745485637, + "pass@16": 0.9183865976477663, + "pass@32": 0.9328827928390062, + "pass@64": 0.9425310412327254, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6829909336419753, + "pass@2": 0.7580878019587827, + "pass@4": 0.8079438594481239, + "pass@8": 0.844415301074487, + "pass@16": 0.8707866725825771, + "pass@32": 0.889554916372676, + "pass@64": 0.9042391369166215, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4906434365781711, + "pass@2": 0.6147872970524704, + "pass@4": 0.712257572615369, + "pass@8": 0.7787509344924866, + "pass@16": 0.825149570594711, + "pass@32": 0.8606246916603065, + "pass@64": 0.8889632425283108, + "pass@128": 0.911504424778761 + }, + "17": { + "pass@1": 0.35130365210843373, + "pass@2": 0.45244795619485817, + "pass@4": 0.5489022199597038, + "pass@8": 0.6336217659934749, + "pass@16": 0.7002461563848493, + "pass@32": 0.7485412599318023, + "pass@64": 0.7885210767175737, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.32331149193548386, + "pass@2": 0.41063071501142995, + "pass@4": 0.4928345900310848, + "pass@8": 0.5661131569413892, + "pass@16": 0.6275198527734406, + "pass@32": 0.6763549924089363, + "pass@64": 0.7168620555395872, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.20834120090634442, + "pass@2": 0.2911352647667531, + "pass@4": 0.37658692710616615, + "pass@8": 0.45385530423580156, + "pass@16": 0.5184556691866746, + "pass@32": 0.5737897113901778, + "pass@64": 0.6263226431161294, + "pass@128": 0.676737160120846 + }, + "20": { + "pass@1": 0.17952327327327328, + "pass@2": 0.2509635619871839, + "pass@4": 0.3270248606311598, + "pass@8": 0.40216650867341297, + "pass@16": 0.46879860714898774, + "pass@32": 0.5219844547616636, + "pass@64": 0.5627529628781948, + "pass@128": 0.5945945945945946 + } + } + } + } + } +} \ No newline at end of file diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..564cc91b952a55d2787d3e29d4e16204d5909c03 --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c16ba96f659e29dace89d5ba4bf6ea69716df3059d39f762cbd9b069be753b12 +size 2646888963 diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/summary.csv b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9d150ee1b4d39defb975d7a8336ca39c64dc373e --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op9-12,0.9937760416666667,0.5772890625,0.7745723684210526,0.0,0.0,0.0,199.80026041666667,354.19277734375,1152000,1280000,2432000 diff --git a/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/summary.json b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..927a9d9e999cb567db069e0df9dfc6b357af758b --- /dev/null +++ b/results/composition/block4/SDAR-100M-block8-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op9-12", + "id_acc": 0.9937760416666667, + "ood_acc": 0.5772890625, + "total_acc": 0.7745723684210526, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.80026041666667, + "ood_avg_resp_len": 354.19277734375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 628343, + "answer_accuracy": 0.7759926790230793, + "avg_response_len": 267.9491656457477, + "resp_tokens_sum": 216965942, + "pass_at_k": { + "pass@1": 0.7759926790230793, + "pass@2": 0.8130554309984241, + "pass@4": 0.8426932765290545, + "pass@8": 0.8662850353724942, + "pass@16": 0.885208594330536, + "pass@32": 0.9002985644575572, + "pass@64": 0.9123666207294034, + "pass@128": 0.92285804615871 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9619976032448377, + "pass@2": 0.9734375362924764, + "pass@4": 0.9813279509862154, + "pass@8": 0.9865472362157778, + "pass@16": 0.9902011494739077, + "pass@32": 0.9925198702388486, + "pass@64": 0.9937310418577298, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9993896484375, + "pass@2": 0.999884657972441, + "pass@4": 0.9999962944085115, + "pass@8": 0.9999999976359266, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9992311507936508, + "pass@2": 0.9998183820772404, + "pass@4": 0.9999906365870932, + "pass@8": 0.9999999824833706, + "pass@16": 0.9999999999999897, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996959206586826, + "pass@2": 0.9999712680937338, + "pass@4": 0.9999997993327181, + "pass@8": 0.9999999999973048, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997688609467456, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999813, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9937650240384616, + "pass@2": 0.9979112248889561, + "pass@4": 0.9995429453529848, + "pass@8": 0.9999526963573945, + "pass@16": 0.9999995305865249, + "pass@32": 0.9999999999899963, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952904929577465, + "pass@2": 0.9969411944105578, + "pass@4": 0.9974237313645655, + "pass@8": 0.9976825868840717, + "pass@16": 0.9981192931869252, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9536782932011332, + "pass@2": 0.9652242588833622, + "pass@4": 0.9721828820618381, + "pass@8": 0.9764794430448108, + "pass@16": 0.9788281054613178, + "pass@32": 0.9806032397021716, + "pass@64": 0.9823113128676721, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.896826447740113, + "pass@2": 0.9179442479647675, + "pass@4": 0.9305790242851, + "pass@8": 0.9397665516265951, + "pass@16": 0.9472254013802787, + "pass@32": 0.9532869258033493, + "pass@64": 0.9573208647827872, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8412029780564263, + "pass@2": 0.8873328468639701, + "pass@4": 0.918331002511833, + "pass@8": 0.9388124431693992, + "pass@16": 0.951719498399668, + "pass@32": 0.9604790149561151, + "pass@64": 0.9673495268907015, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8067805685358256, + "pass@2": 0.8619935395049919, + "pass@4": 0.8958573317938054, + "pass@8": 0.9184229052436736, + "pass@16": 0.9341940408306993, + "pass@32": 0.9454088534696391, + "pass@64": 0.9546836397889742, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7217982297687862, + "pass@2": 0.8086309083564697, + "pass@4": 0.8671244056017556, + "pass@8": 0.9076261577759502, + "pass@16": 0.9361793648177896, + "pass@32": 0.9553006962363504, + "pass@64": 0.9674216402888633, + "pass@128": 0.976878612716763 + }, + "16": { + "pass@1": 0.5188508064516129, + "pass@2": 0.6318909544069088, + "pass@4": 0.7154870514169593, + "pass@8": 0.7761161641465719, + "pass@16": 0.8212375445719823, + "pass@32": 0.8539394390663827, + "pass@64": 0.8752827466227667, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.405301661849711, + "pass@2": 0.513899358813891, + "pass@4": 0.6023861773419938, + "pass@8": 0.6674118975416066, + "pass@16": 0.7150319798410362, + "pass@32": 0.7483390641538125, + "pass@64": 0.7720001182291946, + "pass@128": 0.7947976878612717 + }, + "18": { + "pass@1": 0.28941964285714283, + "pass@2": 0.3808032199100109, + "pass@4": 0.465654920456371, + "pass@8": 0.5423820846954517, + "pass@16": 0.6123445134417186, + "pass@32": 0.6757494775720401, + "pass@64": 0.728451061773362, + "pass@128": 0.7685714285714286 + }, + "19": { + "pass@1": 0.20218656156156156, + "pass@2": 0.27512687392116536, + "pass@4": 0.34533306365983546, + "pass@8": 0.40891840561059223, + "pass@16": 0.4650283703236653, + "pass@32": 0.5150359397845146, + "pass@64": 0.5610112974633917, + "pass@128": 0.6036036036036037 + }, + "20": { + "pass@1": 0.1926614114114114, + "pass@2": 0.268538962190537, + "pass@4": 0.34810443598829427, + "pass@8": 0.4242786966820873, + "pass@16": 0.49075814995394723, + "pass@32": 0.5452179963812089, + "pass@64": 0.5923147933525963, + "pass@128": 0.6396396396396397 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 638598, + "answer_accuracy": 0.7777158028059236, + "avg_response_len": 277.42006771239284, + "resp_tokens_sum": 227795166, + "pass_at_k": { + "pass@1": 0.7777158028059236, + "pass@2": 0.8149754167152435, + "pass@4": 0.8452618992384288, + "pass@8": 0.869452023072591, + "pass@16": 0.8887811131828001, + "pass@32": 0.9042610584313685, + "pass@64": 0.916687088052349, + "pass@128": 0.9270459859703819 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9473015720858896, + "pass@2": 0.9586338673252502, + "pass@4": 0.9689450889190999, + "pass@8": 0.9781974552058535, + "pass@16": 0.9856057187041461, + "pass@32": 0.9914379875697714, + "pass@64": 0.995611394492944, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995244565217392, + "pass@2": 0.9999536403058312, + "pass@4": 0.9999995041380697, + "pass@8": 0.9999999999739078, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986568767908309, + "pass@2": 0.9993968283960923, + "pass@4": 0.9998777572216081, + "pass@8": 0.9999955558687446, + "pass@16": 0.9999999966321708, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959507042253521, + "pass@2": 0.9969051513807253, + "pass@4": 0.9971650308676204, + "pass@8": 0.9971830224753971, + "pass@16": 0.9971830985911607, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9915325598802395, + "pass@2": 0.9940396028808524, + "pass@4": 0.9956864305696818, + "pass@8": 0.9970421805475395, + "pass@16": 0.9984969290242817, + "pass@32": 0.999592837378342, + "pass@64": 0.9999802874493858, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.959972703313253, + "pass@2": 0.9685469239161373, + "pass@4": 0.9733140828217254, + "pass@8": 0.9767761134345652, + "pass@16": 0.9800878380863752, + "pass@32": 0.9835276964020698, + "pass@64": 0.9863941977444657, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9062010188087775, + "pass@2": 0.9236768907511169, + "pass@4": 0.9325341756849359, + "pass@8": 0.9386648112986568, + "pass@16": 0.9453690133693744, + "pass@32": 0.9528166752988761, + "pass@64": 0.96035720610297, + "pass@128": 0.9686520376175548 + }, + "13": { + "pass@1": 0.8154105392156863, + "pass@2": 0.8603254786166442, + "pass@4": 0.8889493970816668, + "pass@8": 0.9097444640506971, + "pass@16": 0.9264317584317059, + "pass@32": 0.9395132059440757, + "pass@64": 0.9505331010668854, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.7924433479532164, + "pass@2": 0.8606995671593685, + "pass@4": 0.9008573744619639, + "pass@8": 0.9240949005993414, + "pass@16": 0.9385674595627927, + "pass@32": 0.9479368741078822, + "pass@64": 0.9556970023204434, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.7134232954545454, + "pass@2": 0.7998765956812215, + "pass@4": 0.8590623126654622, + "pass@8": 0.8993640314330371, + "pass@16": 0.9265664600231758, + "pass@32": 0.944361073070663, + "pass@64": 0.9567259670636806, + "pass@128": 0.9636363636363636 + }, + "16": { + "pass@1": 0.5023593304843305, + "pass@2": 0.6129323978733429, + "pass@4": 0.7012541896472335, + "pass@8": 0.767042798214594, + "pass@16": 0.813623451502168, + "pass@32": 0.8470492421822465, + "pass@64": 0.8704872354663128, + "pass@128": 0.8831908831908832 + }, + "17": { + "pass@1": 0.40945263975155277, + "pass@2": 0.5173417096640096, + "pass@4": 0.6044674413757283, + "pass@8": 0.671494534674276, + "pass@16": 0.7233635296161987, + "pass@32": 0.7620572120609392, + "pass@64": 0.7921207132131408, + "pass@128": 0.8167701863354038 + }, + "18": { + "pass@1": 0.33120404411764703, + "pass@2": 0.4317359744094487, + "pass@4": 0.5253682403670127, + "pass@8": 0.607578615439266, + "pass@16": 0.6789513923043637, + "pass@32": 0.7411219173875988, + "pass@64": 0.7907025142353047, + "pass@128": 0.8294117647058824 + }, + "19": { + "pass@1": 0.21430896577380953, + "pass@2": 0.28969486724315685, + "pass@4": 0.36861701020631393, + "pass@8": 0.44152619961345574, + "pass@16": 0.5023052262297876, + "pass@32": 0.5512420329630123, + "pass@64": 0.5924569517750432, + "pass@128": 0.6309523809523809 + }, + "20": { + "pass@1": 0.18263473053892215, + "pass@2": 0.25554599462492333, + "pass@4": 0.3297973451297629, + "pass@8": 0.397951940647655, + "pass@16": 0.45852535327345006, + "pass@32": 0.5124159362936721, + "pass@64": 0.559076574550227, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 616819, + "answer_accuracy": 0.769915072295894, + "avg_response_len": 298.0402557816744, + "resp_tokens_sum": 238775547, + "pass_at_k": { + "pass@1": 0.769915072295894, + "pass@2": 0.8070560086388907, + "pass@4": 0.8383880848473545, + "pass@8": 0.863916598711563, + "pass@16": 0.8839953800369682, + "pass@32": 0.8996016895462834, + "pass@64": 0.9124875860443917, + "pass@128": 0.9237897427704106 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9641791044776119, + "pass@2": 0.9726069455870255, + "pass@4": 0.9807715807912069, + "pass@8": 0.9873516073772327, + "pass@16": 0.9909816376011114, + "pass@32": 0.992940245540936, + "pass@64": 0.9947799994272432, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995404411764706, + "pass@2": 0.9999829927049559, + "pass@4": 0.9999999765654294, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999260646687698, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930790165165165, + "pass@2": 0.9959048615544679, + "pass@4": 0.9976245238489334, + "pass@8": 0.9985557744861908, + "pass@16": 0.9993523742551476, + "pass@32": 0.9998911703937988, + "pass@64": 0.9999990823608353, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925140675241158, + "pass@2": 0.9940972573866369, + "pass@4": 0.9955341559997924, + "pass@8": 0.9966746145755141, + "pass@16": 0.9974756056363967, + "pass@32": 0.998195401826233, + "pass@64": 0.9992024710718075, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9621527777777777, + "pass@2": 0.9740993313335832, + "pass@4": 0.980316312841847, + "pass@8": 0.9843892772286572, + "pass@16": 0.9873977583523199, + "pass@32": 0.9893976342731529, + "pass@64": 0.9912758756244631, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9057960626911316, + "pass@2": 0.9306218997567968, + "pass@4": 0.9471255929591371, + "pass@8": 0.957859276936109, + "pass@16": 0.9657104328023974, + "pass@32": 0.972663554714854, + "pass@64": 0.9783440153777522, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7977189429012346, + "pass@2": 0.8490566825848157, + "pass@4": 0.8812317392617589, + "pass@8": 0.9021572476122781, + "pass@16": 0.9167989658985097, + "pass@32": 0.9276796213050199, + "pass@64": 0.9364064746699794, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7648831602373887, + "pass@2": 0.8279475717890604, + "pass@4": 0.8702395625064661, + "pass@8": 0.8985536745485637, + "pass@16": 0.9183865976477663, + "pass@32": 0.9328827928390062, + "pass@64": 0.9425310412327254, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6829909336419753, + "pass@2": 0.7580878019587827, + "pass@4": 0.8079438594481239, + "pass@8": 0.844415301074487, + "pass@16": 0.8707866725825771, + "pass@32": 0.889554916372676, + "pass@64": 0.9042391369166215, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4906434365781711, + "pass@2": 0.6147872970524704, + "pass@4": 0.712257572615369, + "pass@8": 0.7787509344924866, + "pass@16": 0.825149570594711, + "pass@32": 0.8606246916603065, + "pass@64": 0.8889632425283108, + "pass@128": 0.911504424778761 + }, + "17": { + "pass@1": 0.35130365210843373, + "pass@2": 0.45244795619485817, + "pass@4": 0.5489022199597038, + "pass@8": 0.6336217659934749, + "pass@16": 0.7002461563848493, + "pass@32": 0.7485412599318023, + "pass@64": 0.7885210767175737, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.32331149193548386, + "pass@2": 0.41063071501142995, + "pass@4": 0.4928345900310848, + "pass@8": 0.5661131569413892, + "pass@16": 0.6275198527734406, + "pass@32": 0.6763549924089363, + "pass@64": 0.7168620555395872, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.20834120090634442, + "pass@2": 0.2911352647667531, + "pass@4": 0.37658692710616615, + "pass@8": 0.45385530423580156, + "pass@16": 0.5184556691866746, + "pass@32": 0.5737897113901778, + "pass@64": 0.6263226431161294, + "pass@128": 0.676737160120846 + }, + "20": { + "pass@1": 0.17952327327327328, + "pass@2": 0.2509635619871839, + "pass@4": 0.3270248606311598, + "pass@8": 0.40216650867341297, + "pass@16": 0.46879860714898774, + "pass@32": 0.5219844547616636, + "pass@64": 0.5627529628781948, + "pass@128": 0.5945945945945946 + } + } + } + } + } +] \ No newline at end of file diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a42cd90cd56e5cbc0fe1fe1f5236440e71175934 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ac20fa00dca98c24be014d8a55c2bd043d06cdaf0417ce56512f19cb8c7f649 +size 1644144113 diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_metrics.json b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e9980f4e8ec0c754d21ccd9ae0327a5289013b46 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2", + "id": { + "answer_accuracy": 0.36663975694444445, + "avg_response_len": 166.67154947916666, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.36663975694444445, + "pass@2": 0.40109596456693464, + "pass@4": 0.43236788167104534, + "pass@8": 0.46500848836997294, + "pass@16": 0.5011790198093209, + "pass@32": 0.5407341668737213, + "pass@64": 0.5821693705055739, + "pass@128": 0.6231111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.185359375, + "pass@2": 0.22665095964566928, + "pass@4": 0.26727793813273304, + "pass@8": 0.30954397222044167, + "pass@16": 0.35545953862561114, + "pass@32": 0.406521544874947, + "pass@64": 0.46236825402274806, + "pass@128": 0.518 + }, + "2": { + "pass@1": 0.9783828125, + "pass@2": 0.985677288385827, + "pass@4": 0.9900933495500566, + "pass@8": 0.9934069664363652, + "pass@16": 0.9959995639693671, + "pass@32": 0.9980012422309414, + "pass@64": 0.9993614077206364, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.8024375, + "pass@2": 0.8637383120078743, + "pass@4": 0.9026073085864265, + "pass@8": 0.9289196725664263, + "pass@16": 0.9472079819152873, + "pass@32": 0.9598822779486047, + "pass@64": 0.9685287078858967, + "pass@128": 0.974 + }, + "4": { + "pass@1": 0.2260703125, + "pass@2": 0.25332062007874023, + "pass@4": 0.28066603299587534, + "pass@8": 0.3103555783316386, + "pass@16": 0.34315392227388125, + "pass@32": 0.3783126435894053, + "pass@64": 0.41367619377252585, + "pass@128": 0.448 + }, + "5": { + "pass@1": 0.2620703125, + "pass@2": 0.29352362204724425, + "pass@4": 0.3214579548181478, + "pass@8": 0.3493021403707522, + "pass@16": 0.3807327379659776, + "pass@32": 0.4177264064718202, + "pass@64": 0.4594949425831975, + "pass@128": 0.501 + }, + "6": { + "pass@1": 0.257390625, + "pass@2": 0.2899992618110238, + "pass@4": 0.3218167166291711, + "pass@8": 0.3592720826099001, + "pass@16": 0.4044761271292741, + "pass@32": 0.456011217362287, + "pass@64": 0.5093909478948577, + "pass@128": 0.557 + }, + "7": { + "pass@1": 0.2025703125, + "pass@2": 0.2358702017716536, + "pass@4": 0.27075396878515157, + "pass@8": 0.3118831160712313, + "pass@16": 0.36057097564467266, + "pass@32": 0.41305512983021886, + "pass@64": 0.4655551878213001, + "pass@128": 0.516 + }, + "8": { + "pass@1": 0.200203125, + "pass@2": 0.23747268700787386, + "pass@4": 0.27527332133483295, + "pass@8": 0.3189794590222602, + "pass@16": 0.37092671699208357, + "pass@32": 0.4298585350973072, + "pass@64": 0.4941520825565573, + "pass@128": 0.566 + }, + "9": { + "pass@1": 0.1852734375, + "pass@2": 0.22361072834645673, + "pass@4": 0.26136434420697385, + "pass@8": 0.30341340770066794, + "pass@16": 0.35208361376766595, + "pass@32": 0.40723850445797893, + "pass@64": 0.4669966102924466, + "pass@128": 0.528 + } + }, + "per_op_accuracy": { + "10": 0.185359375, + "2": 0.9783828125, + "3": 0.8024375, + "4": 0.2260703125, + "5": 0.2620703125, + "6": 0.257390625, + "7": 0.2025703125, + "8": 0.200203125, + "9": 0.1852734375 + }, + "per_op_avg_response_len": { + "10": 228.4636953125, + "2": 113.0011171875, + "3": 127.0326484375, + "4": 130.8636875, + "5": 152.778015625, + "6": 177.1742734375, + "7": 165.6185703125, + "8": 194.39275, + "9": 210.7191875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 228.4636953125, + "2": 113.0011171875, + "3": 127.0326484375, + "4": 130.8636875, + "5": 152.778015625, + "6": 177.1742734375, + "7": 165.6185703125, + "8": 194.39275, + "9": 210.7191875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 422369, + "answer_accuracy": 0.36663975694444445, + "avg_response_len": 166.67154947916666, + "resp_tokens_sum": 192005625, + "pass_at_k": { + "pass@1": 0.36663975694444445, + "pass@2": 0.40109596456693464, + "pass@4": 0.43236788167104534, + "pass@8": 0.46500848836997294, + "pass@16": 0.5011790198093209, + "pass@32": 0.5407341668737213, + "pass@64": 0.5821693705055739, + "pass@128": 0.6231111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.185359375, + "pass@2": 0.22665095964566928, + "pass@4": 0.26727793813273304, + "pass@8": 0.30954397222044167, + "pass@16": 0.35545953862561114, + "pass@32": 0.406521544874947, + "pass@64": 0.46236825402274806, + "pass@128": 0.518 + }, + "2": { + "pass@1": 0.9783828125, + "pass@2": 0.985677288385827, + "pass@4": 0.9900933495500566, + "pass@8": 0.9934069664363652, + "pass@16": 0.9959995639693671, + "pass@32": 0.9980012422309414, + "pass@64": 0.9993614077206364, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.8024375, + "pass@2": 0.8637383120078743, + "pass@4": 0.9026073085864265, + "pass@8": 0.9289196725664263, + "pass@16": 0.9472079819152873, + "pass@32": 0.9598822779486047, + "pass@64": 0.9685287078858967, + "pass@128": 0.974 + }, + "4": { + "pass@1": 0.2260703125, + "pass@2": 0.25332062007874023, + "pass@4": 0.28066603299587534, + "pass@8": 0.3103555783316386, + "pass@16": 0.34315392227388125, + "pass@32": 0.3783126435894053, + "pass@64": 0.41367619377252585, + "pass@128": 0.448 + }, + "5": { + "pass@1": 0.2620703125, + "pass@2": 0.29352362204724425, + "pass@4": 0.3214579548181478, + "pass@8": 0.3493021403707522, + "pass@16": 0.3807327379659776, + "pass@32": 0.4177264064718202, + "pass@64": 0.4594949425831975, + "pass@128": 0.501 + }, + "6": { + "pass@1": 0.257390625, + "pass@2": 0.2899992618110238, + "pass@4": 0.3218167166291711, + "pass@8": 0.3592720826099001, + "pass@16": 0.4044761271292741, + "pass@32": 0.456011217362287, + "pass@64": 0.5093909478948577, + "pass@128": 0.557 + }, + "7": { + "pass@1": 0.2025703125, + "pass@2": 0.2358702017716536, + "pass@4": 0.27075396878515157, + "pass@8": 0.3118831160712313, + "pass@16": 0.36057097564467266, + "pass@32": 0.41305512983021886, + "pass@64": 0.4655551878213001, + "pass@128": 0.516 + }, + "8": { + "pass@1": 0.200203125, + "pass@2": 0.23747268700787386, + "pass@4": 0.27527332133483295, + "pass@8": 0.3189794590222602, + "pass@16": 0.37092671699208357, + "pass@32": 0.4298585350973072, + "pass@64": 0.4941520825565573, + "pass@128": 0.566 + }, + "9": { + "pass@1": 0.1852734375, + "pass@2": 0.22361072834645673, + "pass@4": 0.26136434420697385, + "pass@8": 0.30341340770066794, + "pass@16": 0.35208361376766595, + "pass@32": 0.40723850445797893, + "pass@64": 0.4669966102924466, + "pass@128": 0.528 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.1485921875, + "avg_response_len": 288.48673046875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.1485921875, + "pass@2": 0.18976325049212697, + "pass@4": 0.2295522008717676, + "pass@8": 0.2688561758597625, + "pass@16": 0.30867647711095475, + "pass@32": 0.34965291661457315, + "pass@64": 0.3929946420178624, + "pass@128": 0.4394 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.181328125, + "pass@2": 0.22308956692913356, + "pass@4": 0.2620878564866889, + "pass@8": 0.30073618152874915, + "pass@16": 0.34229874395712756, + "pass@32": 0.38838180923282895, + "pass@64": 0.4374700679217919, + "pass@128": 0.487 + }, + "12": { + "pass@1": 0.1834296875, + "pass@2": 0.22520152559055115, + "pass@4": 0.2631668171166101, + "pass@8": 0.2993326883550847, + "pass@16": 0.3351238797250115, + "pass@32": 0.37152423224352726, + "pass@64": 0.4098912760329702, + "pass@128": 0.451 + }, + "13": { + "pass@1": 0.149203125, + "pass@2": 0.1888870570866141, + "pass@4": 0.2266780743344578, + "pass@8": 0.26277767341577507, + "pass@16": 0.29816678392720575, + "pass@32": 0.335296891994066, + "pass@64": 0.3763319045836284, + "pass@128": 0.422 + }, + "14": { + "pass@1": 0.1473671875, + "pass@2": 0.19373252952755887, + "pass@4": 0.2384486215785524, + "pass@8": 0.28300815279693214, + "pass@16": 0.32785614148323466, + "pass@32": 0.37277487753458227, + "pass@64": 0.4184688571757885, + "pass@128": 0.466 + }, + "15": { + "pass@1": 0.1339375, + "pass@2": 0.17233956692913383, + "pass@4": 0.21110731636670393, + "pass@8": 0.2508732570464338, + "pass@16": 0.29181007808554676, + "pass@32": 0.3342102860408758, + "pass@64": 0.3805825749395631, + "pass@128": 0.43 + }, + "16": { + "pass@1": 0.1640390625, + "pass@2": 0.20756471456692913, + "pass@4": 0.2486257512185974, + "pass@8": 0.28807563999679026, + "pass@16": 0.32744718667297784, + "pass@32": 0.36867748793616795, + "pass@64": 0.41376173730866433, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.1465625, + "pass@2": 0.19127854330708668, + "pass@4": 0.23576637992125982, + "pass@8": 0.27938039291798156, + "pass@16": 0.3211327072075191, + "pass@32": 0.3608730422253764, + "pass@64": 0.40151002182685186, + "pass@128": 0.443 + }, + "18": { + "pass@1": 0.1308125, + "pass@2": 0.17329847440944876, + "pass@4": 0.2158036766966628, + "pass@8": 0.2579953436262119, + "pass@16": 0.3009428053842477, + "pass@32": 0.3455988365948172, + "pass@64": 0.3931462119418665, + "pass@128": 0.445 + }, + "19": { + "pass@1": 0.1276875, + "pass@2": 0.1649736712598425, + "pass@4": 0.20048644581927239, + "pass@8": 0.23576637534301864, + "pass@16": 0.2727791010100153, + "pass@32": 0.31083187841575016, + "pass@64": 0.34949288202114465, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.1215546875, + "pass@2": 0.15726685531496062, + "pass@4": 0.1933510691788525, + "pass@8": 0.23061605357061016, + "pass@16": 0.26920734365662685, + "pass@32": 0.3083598239277578, + "pass@64": 0.3492908864263997, + "pass@128": 0.397 + } + }, + "per_op_accuracy": { + "11": 0.181328125, + "12": 0.1834296875, + "13": 0.149203125, + "14": 0.1473671875, + "15": 0.1339375, + "16": 0.1640390625, + "17": 0.1465625, + "18": 0.1308125, + "19": 0.1276875, + "20": 0.1215546875 + }, + "per_op_avg_response_len": { + "11": 234.1059921875, + "12": 231.47190625, + "13": 245.08246875, + "14": 285.1685078125, + "15": 294.4890234375, + "16": 304.7250703125, + "17": 318.7973046875, + "18": 328.1741875, + "19": 318.3943828125, + "20": 324.4584609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 234.1059921875, + "12": 231.47190625, + "13": 245.08246875, + "14": 285.1685078125, + "15": 294.4890234375, + "16": 304.7250703125, + "17": 318.7973046875, + "18": 328.1741875, + "19": 318.3943828125, + "20": 324.4584609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 190198, + "answer_accuracy": 0.1485921875, + "avg_response_len": 288.48673046875, + "resp_tokens_sum": 369263015, + "pass_at_k": { + "pass@1": 0.1485921875, + "pass@2": 0.18976325049212697, + "pass@4": 0.2295522008717676, + "pass@8": 0.2688561758597625, + "pass@16": 0.30867647711095475, + "pass@32": 0.34965291661457315, + "pass@64": 0.3929946420178624, + "pass@128": 0.4394 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.181328125, + "pass@2": 0.22308956692913356, + "pass@4": 0.2620878564866889, + "pass@8": 0.30073618152874915, + "pass@16": 0.34229874395712756, + "pass@32": 0.38838180923282895, + "pass@64": 0.4374700679217919, + "pass@128": 0.487 + }, + "12": { + "pass@1": 0.1834296875, + "pass@2": 0.22520152559055115, + "pass@4": 0.2631668171166101, + "pass@8": 0.2993326883550847, + "pass@16": 0.3351238797250115, + "pass@32": 0.37152423224352726, + "pass@64": 0.4098912760329702, + "pass@128": 0.451 + }, + "13": { + "pass@1": 0.149203125, + "pass@2": 0.1888870570866141, + "pass@4": 0.2266780743344578, + "pass@8": 0.26277767341577507, + "pass@16": 0.29816678392720575, + "pass@32": 0.335296891994066, + "pass@64": 0.3763319045836284, + "pass@128": 0.422 + }, + "14": { + "pass@1": 0.1473671875, + "pass@2": 0.19373252952755887, + "pass@4": 0.2384486215785524, + "pass@8": 0.28300815279693214, + "pass@16": 0.32785614148323466, + "pass@32": 0.37277487753458227, + "pass@64": 0.4184688571757885, + "pass@128": 0.466 + }, + "15": { + "pass@1": 0.1339375, + "pass@2": 0.17233956692913383, + "pass@4": 0.21110731636670393, + "pass@8": 0.2508732570464338, + "pass@16": 0.29181007808554676, + "pass@32": 0.3342102860408758, + "pass@64": 0.3805825749395631, + "pass@128": 0.43 + }, + "16": { + "pass@1": 0.1640390625, + "pass@2": 0.20756471456692913, + "pass@4": 0.2486257512185974, + "pass@8": 0.28807563999679026, + "pass@16": 0.32744718667297784, + "pass@32": 0.36867748793616795, + "pass@64": 0.41376173730866433, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.1465625, + "pass@2": 0.19127854330708668, + "pass@4": 0.23576637992125982, + "pass@8": 0.27938039291798156, + "pass@16": 0.3211327072075191, + "pass@32": 0.3608730422253764, + "pass@64": 0.40151002182685186, + "pass@128": 0.443 + }, + "18": { + "pass@1": 0.1308125, + "pass@2": 0.17329847440944876, + "pass@4": 0.2158036766966628, + "pass@8": 0.2579953436262119, + "pass@16": 0.3009428053842477, + "pass@32": 0.3455988365948172, + "pass@64": 0.3931462119418665, + "pass@128": 0.445 + }, + "19": { + "pass@1": 0.1276875, + "pass@2": 0.1649736712598425, + "pass@4": 0.20048644581927239, + "pass@8": 0.23576637534301864, + "pass@16": 0.2727791010100153, + "pass@32": 0.31083187841575016, + "pass@64": 0.34949288202114465, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.1215546875, + "pass@2": 0.15726685531496062, + "pass@4": 0.1933510691788525, + "pass@8": 0.23061605357061016, + "pass@16": 0.26920734365662685, + "pass@32": 0.3083598239277578, + "pass@64": 0.3492908864263997, + "pass@128": 0.397 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.25187787828947367, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.25187787828947367, + "pass@2": 0.2898682203170242, + "pass@4": 0.3256227865135233, + "pass@8": 0.36177042915407004, + "pass@16": 0.39986189207334566, + "pass@32": 0.4401650877899612, + "pass@64": 0.48260372393310047, + "pass@128": 0.5264210526315789 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.185359375, + "pass@2": 0.22665095964566928, + "pass@4": 0.26727793813273304, + "pass@8": 0.30954397222044167, + "pass@16": 0.35545953862561114, + "pass@32": 0.406521544874947, + "pass@64": 0.46236825402274806, + "pass@128": 0.518 + }, + "2": { + "pass@1": 0.9783828125, + "pass@2": 0.985677288385827, + "pass@4": 0.9900933495500566, + "pass@8": 0.9934069664363652, + "pass@16": 0.9959995639693671, + "pass@32": 0.9980012422309414, + "pass@64": 0.9993614077206364, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.8024375, + "pass@2": 0.8637383120078743, + "pass@4": 0.9026073085864265, + "pass@8": 0.9289196725664263, + "pass@16": 0.9472079819152873, + "pass@32": 0.9598822779486047, + "pass@64": 0.9685287078858967, + "pass@128": 0.974 + }, + "4": { + "pass@1": 0.2260703125, + "pass@2": 0.25332062007874023, + "pass@4": 0.28066603299587534, + "pass@8": 0.3103555783316386, + "pass@16": 0.34315392227388125, + "pass@32": 0.3783126435894053, + "pass@64": 0.41367619377252585, + "pass@128": 0.448 + }, + "5": { + "pass@1": 0.2620703125, + "pass@2": 0.29352362204724425, + "pass@4": 0.3214579548181478, + "pass@8": 0.3493021403707522, + "pass@16": 0.3807327379659776, + "pass@32": 0.4177264064718202, + "pass@64": 0.4594949425831975, + "pass@128": 0.501 + }, + "6": { + "pass@1": 0.257390625, + "pass@2": 0.2899992618110238, + "pass@4": 0.3218167166291711, + "pass@8": 0.3592720826099001, + "pass@16": 0.4044761271292741, + "pass@32": 0.456011217362287, + "pass@64": 0.5093909478948577, + "pass@128": 0.557 + }, + "7": { + "pass@1": 0.2025703125, + "pass@2": 0.2358702017716536, + "pass@4": 0.27075396878515157, + "pass@8": 0.3118831160712313, + "pass@16": 0.36057097564467266, + "pass@32": 0.41305512983021886, + "pass@64": 0.4655551878213001, + "pass@128": 0.516 + }, + "8": { + "pass@1": 0.200203125, + "pass@2": 0.23747268700787386, + "pass@4": 0.27527332133483295, + "pass@8": 0.3189794590222602, + "pass@16": 0.37092671699208357, + "pass@32": 0.4298585350973072, + "pass@64": 0.4941520825565573, + "pass@128": 0.566 + }, + "9": { + "pass@1": 0.1852734375, + "pass@2": 0.22361072834645673, + "pass@4": 0.26136434420697385, + "pass@8": 0.30341340770066794, + "pass@16": 0.35208361376766595, + "pass@32": 0.40723850445797893, + "pass@64": 0.4669966102924466, + "pass@128": 0.528 + }, + "11": { + "pass@1": 0.181328125, + "pass@2": 0.22308956692913356, + "pass@4": 0.2620878564866889, + "pass@8": 0.30073618152874915, + "pass@16": 0.34229874395712756, + "pass@32": 0.38838180923282895, + "pass@64": 0.4374700679217919, + "pass@128": 0.487 + }, + "12": { + "pass@1": 0.1834296875, + "pass@2": 0.22520152559055115, + "pass@4": 0.2631668171166101, + "pass@8": 0.2993326883550847, + "pass@16": 0.3351238797250115, + "pass@32": 0.37152423224352726, + "pass@64": 0.4098912760329702, + "pass@128": 0.451 + }, + "13": { + "pass@1": 0.149203125, + "pass@2": 0.1888870570866141, + "pass@4": 0.2266780743344578, + "pass@8": 0.26277767341577507, + "pass@16": 0.29816678392720575, + "pass@32": 0.335296891994066, + "pass@64": 0.3763319045836284, + "pass@128": 0.422 + }, + "14": { + "pass@1": 0.1473671875, + "pass@2": 0.19373252952755887, + "pass@4": 0.2384486215785524, + "pass@8": 0.28300815279693214, + "pass@16": 0.32785614148323466, + "pass@32": 0.37277487753458227, + "pass@64": 0.4184688571757885, + "pass@128": 0.466 + }, + "15": { + "pass@1": 0.1339375, + "pass@2": 0.17233956692913383, + "pass@4": 0.21110731636670393, + "pass@8": 0.2508732570464338, + "pass@16": 0.29181007808554676, + "pass@32": 0.3342102860408758, + "pass@64": 0.3805825749395631, + "pass@128": 0.43 + }, + "16": { + "pass@1": 0.1640390625, + "pass@2": 0.20756471456692913, + "pass@4": 0.2486257512185974, + "pass@8": 0.28807563999679026, + "pass@16": 0.32744718667297784, + "pass@32": 0.36867748793616795, + "pass@64": 0.41376173730866433, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.1465625, + "pass@2": 0.19127854330708668, + "pass@4": 0.23576637992125982, + "pass@8": 0.27938039291798156, + "pass@16": 0.3211327072075191, + "pass@32": 0.3608730422253764, + "pass@64": 0.40151002182685186, + "pass@128": 0.443 + }, + "18": { + "pass@1": 0.1308125, + "pass@2": 0.17329847440944876, + "pass@4": 0.2158036766966628, + "pass@8": 0.2579953436262119, + "pass@16": 0.3009428053842477, + "pass@32": 0.3455988365948172, + "pass@64": 0.3931462119418665, + "pass@128": 0.445 + }, + "19": { + "pass@1": 0.1276875, + "pass@2": 0.1649736712598425, + "pass@4": 0.20048644581927239, + "pass@8": 0.23576637534301864, + "pass@16": 0.2727791010100153, + "pass@32": 0.31083187841575016, + "pass@64": 0.34949288202114465, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.1215546875, + "pass@2": 0.15726685531496062, + "pass@4": 0.1933510691788525, + "pass@8": 0.23061605357061016, + "pass@16": 0.26920734365662685, + "pass@32": 0.3083598239277578, + "pass@64": 0.3492908864263997, + "pass@128": 0.397 + } + }, + "per_op_accuracy": { + "10": 0.185359375, + "2": 0.9783828125, + "3": 0.8024375, + "4": 0.2260703125, + "5": 0.2620703125, + "6": 0.257390625, + "7": 0.2025703125, + "8": 0.200203125, + "9": 0.1852734375, + "11": 0.181328125, + "12": 0.1834296875, + "13": 0.149203125, + "14": 0.1473671875, + "15": 0.1339375, + "16": 0.1640390625, + "17": 0.1465625, + "18": 0.1308125, + "19": 0.1276875, + "20": 0.1215546875 + }, + "per_op_avg_response_len": { + "10": 228.4636953125, + "2": 113.0011171875, + "3": 127.0326484375, + "4": 130.8636875, + "5": 152.778015625, + "6": 177.1742734375, + "7": 165.6185703125, + "8": 194.39275, + "9": 210.7191875, + "11": 234.1059921875, + "12": 231.47190625, + "13": 245.08246875, + "14": 285.1685078125, + "15": 294.4890234375, + "16": 304.7250703125, + "17": 318.7973046875, + "18": 328.1741875, + "19": 318.3943828125, + "20": 324.4584609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 228.4636953125, + "2": 113.0011171875, + "3": 127.0326484375, + "4": 130.8636875, + "5": 152.778015625, + "6": 177.1742734375, + "7": 165.6185703125, + "8": 194.39275, + "9": 210.7191875, + "11": 234.1059921875, + "12": 231.47190625, + "13": 245.08246875, + "14": 285.1685078125, + "15": 294.4890234375, + "16": 304.7250703125, + "17": 318.7973046875, + "18": 328.1741875, + "19": 318.3943828125, + "20": 324.4584609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 612567, + "answer_accuracy": 0.25187787828947367, + "avg_response_len": 230.78480263157894, + "resp_tokens_sum": 561268640, + "pass_at_k": { + "pass@1": 0.25187787828947367, + "pass@2": 0.2898682203170242, + "pass@4": 0.3256227865135233, + "pass@8": 0.36177042915407004, + "pass@16": 0.39986189207334566, + "pass@32": 0.4401650877899612, + "pass@64": 0.48260372393310047, + "pass@128": 0.5264210526315789 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.185359375, + "pass@2": 0.22665095964566928, + "pass@4": 0.26727793813273304, + "pass@8": 0.30954397222044167, + "pass@16": 0.35545953862561114, + "pass@32": 0.406521544874947, + "pass@64": 0.46236825402274806, + "pass@128": 0.518 + }, + "2": { + "pass@1": 0.9783828125, + "pass@2": 0.985677288385827, + "pass@4": 0.9900933495500566, + "pass@8": 0.9934069664363652, + "pass@16": 0.9959995639693671, + "pass@32": 0.9980012422309414, + "pass@64": 0.9993614077206364, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.8024375, + "pass@2": 0.8637383120078743, + "pass@4": 0.9026073085864265, + "pass@8": 0.9289196725664263, + "pass@16": 0.9472079819152873, + "pass@32": 0.9598822779486047, + "pass@64": 0.9685287078858967, + "pass@128": 0.974 + }, + "4": { + "pass@1": 0.2260703125, + "pass@2": 0.25332062007874023, + "pass@4": 0.28066603299587534, + "pass@8": 0.3103555783316386, + "pass@16": 0.34315392227388125, + "pass@32": 0.3783126435894053, + "pass@64": 0.41367619377252585, + "pass@128": 0.448 + }, + "5": { + "pass@1": 0.2620703125, + "pass@2": 0.29352362204724425, + "pass@4": 0.3214579548181478, + "pass@8": 0.3493021403707522, + "pass@16": 0.3807327379659776, + "pass@32": 0.4177264064718202, + "pass@64": 0.4594949425831975, + "pass@128": 0.501 + }, + "6": { + "pass@1": 0.257390625, + "pass@2": 0.2899992618110238, + "pass@4": 0.3218167166291711, + "pass@8": 0.3592720826099001, + "pass@16": 0.4044761271292741, + "pass@32": 0.456011217362287, + "pass@64": 0.5093909478948577, + "pass@128": 0.557 + }, + "7": { + "pass@1": 0.2025703125, + "pass@2": 0.2358702017716536, + "pass@4": 0.27075396878515157, + "pass@8": 0.3118831160712313, + "pass@16": 0.36057097564467266, + "pass@32": 0.41305512983021886, + "pass@64": 0.4655551878213001, + "pass@128": 0.516 + }, + "8": { + "pass@1": 0.200203125, + "pass@2": 0.23747268700787386, + "pass@4": 0.27527332133483295, + "pass@8": 0.3189794590222602, + "pass@16": 0.37092671699208357, + "pass@32": 0.4298585350973072, + "pass@64": 0.4941520825565573, + "pass@128": 0.566 + }, + "9": { + "pass@1": 0.1852734375, + "pass@2": 0.22361072834645673, + "pass@4": 0.26136434420697385, + "pass@8": 0.30341340770066794, + "pass@16": 0.35208361376766595, + "pass@32": 0.40723850445797893, + "pass@64": 0.4669966102924466, + "pass@128": 0.528 + }, + "11": { + "pass@1": 0.181328125, + "pass@2": 0.22308956692913356, + "pass@4": 0.2620878564866889, + "pass@8": 0.30073618152874915, + "pass@16": 0.34229874395712756, + "pass@32": 0.38838180923282895, + "pass@64": 0.4374700679217919, + "pass@128": 0.487 + }, + "12": { + "pass@1": 0.1834296875, + "pass@2": 0.22520152559055115, + "pass@4": 0.2631668171166101, + "pass@8": 0.2993326883550847, + "pass@16": 0.3351238797250115, + "pass@32": 0.37152423224352726, + "pass@64": 0.4098912760329702, + "pass@128": 0.451 + }, + "13": { + "pass@1": 0.149203125, + "pass@2": 0.1888870570866141, + "pass@4": 0.2266780743344578, + "pass@8": 0.26277767341577507, + "pass@16": 0.29816678392720575, + "pass@32": 0.335296891994066, + "pass@64": 0.3763319045836284, + "pass@128": 0.422 + }, + "14": { + "pass@1": 0.1473671875, + "pass@2": 0.19373252952755887, + "pass@4": 0.2384486215785524, + "pass@8": 0.28300815279693214, + "pass@16": 0.32785614148323466, + "pass@32": 0.37277487753458227, + "pass@64": 0.4184688571757885, + "pass@128": 0.466 + }, + "15": { + "pass@1": 0.1339375, + "pass@2": 0.17233956692913383, + "pass@4": 0.21110731636670393, + "pass@8": 0.2508732570464338, + "pass@16": 0.29181007808554676, + "pass@32": 0.3342102860408758, + "pass@64": 0.3805825749395631, + "pass@128": 0.43 + }, + "16": { + "pass@1": 0.1640390625, + "pass@2": 0.20756471456692913, + "pass@4": 0.2486257512185974, + "pass@8": 0.28807563999679026, + "pass@16": 0.32744718667297784, + "pass@32": 0.36867748793616795, + "pass@64": 0.41376173730866433, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.1465625, + "pass@2": 0.19127854330708668, + "pass@4": 0.23576637992125982, + "pass@8": 0.27938039291798156, + "pass@16": 0.3211327072075191, + "pass@32": 0.3608730422253764, + "pass@64": 0.40151002182685186, + "pass@128": 0.443 + }, + "18": { + "pass@1": 0.1308125, + "pass@2": 0.17329847440944876, + "pass@4": 0.2158036766966628, + "pass@8": 0.2579953436262119, + "pass@16": 0.3009428053842477, + "pass@32": 0.3455988365948172, + "pass@64": 0.3931462119418665, + "pass@128": 0.445 + }, + "19": { + "pass@1": 0.1276875, + "pass@2": 0.1649736712598425, + "pass@4": 0.20048644581927239, + "pass@8": 0.23576637534301864, + "pass@16": 0.2727791010100153, + "pass@32": 0.31083187841575016, + "pass@64": 0.34949288202114465, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.1215546875, + "pass@2": 0.15726685531496062, + "pass@4": 0.1933510691788525, + "pass@8": 0.23061605357061016, + "pass@16": 0.26920734365662685, + "pass@32": 0.3083598239277578, + "pass@64": 0.3492908864263997, + "pass@128": 0.397 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..70f354cbbaf64fe5d93e2df132a00c32c3766d4a --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b0b2e8abd5e6a7060e6ba227557ea62930b8e8d99b716158b7ab4f964f240e6 +size 2462388951 diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl.graph_results.jsonl b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..268101e5a613fe7d0ba63de6a2dc807165b4ed8a --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:109e03a3bca7870e1f52a5396cebd4d454df3b9d30010a28220a21d2c86aac96 +size 401532465 diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl.graph_results.jsonl b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7122440d4738bff20a97107ae2cc3f3d22c831e7 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26649ff5b806a17fca66fe058b6e24dfe37e0676f72a98dd4cc9a12c224f0fe0 +size 458703992 diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..33854622426b83fb7e3bc6f7617230a4a33b9708 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/graphs", "generated_path": "results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2", "results_dir": "results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs", "per_file": [{"file": "results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 101438, "value_mismatch": 559968, "dependency_mismatch": 688956, "answer_mismatch": 729639, "extra_nodes": 676099, "missing_nodes": 1047801, "perfect_with_extra": 6, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "value_mismatch": 559968, "dependency_mismatch": 688956, "answer_mismatch": 729639, "missing_nodes": 1047801, "extra_nodes": 676099, "perfect": 101438, "perfect_with_extra": 6}}, "results_path": "results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 889572, "dependency_mismatch": 945566, "answer_mismatch": 1089806, "extra_nodes": 1076755, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 889572, "dependency_mismatch": 945566, "answer_mismatch": 1089806, "extra_nodes": 1076755, "missing_nodes": 1280000}}, "results_path": "results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 101438, "value_mismatch": 1449540, "dependency_mismatch": 1634522, "answer_mismatch": 1819445, "extra_nodes": 1752854, "missing_nodes": 2327801, "perfect_with_extra": 6, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "value_mismatch": 1449540, "dependency_mismatch": 1634522, "answer_mismatch": 1819445, "missing_nodes": 2327801, "extra_nodes": 1752854, "perfect": 101438, "perfect_with_extra": 6}}}} diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/results.json b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..c8740b93d72a948b3022beb0d804e25b7e3bafd8 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.811, + "pass@2": 0.871, + "pass@4": 0.903, + "pass@8": 0.939, + "pass@16": 0.951, + "pass@32": 0.96, + "pass@64": 0.974, + "pass@128": 0.984 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.002, + "pass@128": 0.003 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.001, + "pass@128": 0.001 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.811, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.871, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.903, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.939, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.951, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.96, + "3": 0.001, + "4": 0.001, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.974, + "3": 0.002, + "4": 0.001, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.984, + "3": 0.003, + "4": 0.001, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.09011111111111111, + "pass@2": 0.09677777777777778, + "pass@4": 0.10033333333333333, + "pass@8": 0.10433333333333333, + "pass@16": 0.10566666666666667, + "pass@32": 0.1068888888888889, + "pass@64": 0.10855555555555556, + "pass@128": 0.10977777777777778 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.811, + "pass@2": 0.871, + "pass@4": 0.903, + "pass@8": 0.939, + "pass@16": 0.951, + "pass@32": 0.96, + "pass@64": 0.974, + "pass@128": 0.984 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.002, + "pass@128": 0.003 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.001, + "pass@128": 0.001 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.811, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.871, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.903, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.939, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.951, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.96, + "3": 0.001, + "4": 0.001, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.974, + "3": 0.002, + "4": 0.001, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.984, + "3": 0.003, + "4": 0.001, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.09011111111111111, + "pass@2": 0.09677777777777778, + "pass@4": 0.10033333333333333, + "pass@8": 0.10433333333333333, + "pass@16": 0.10566666666666667, + "pass@32": 0.1068888888888889, + "pass@64": 0.10855555555555556, + "pass@128": 0.10977777777777778 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/summary.csv b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..f01900b1935d852011f43490fa05ab23db08f11c --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2,0.36663975694444445,0.1485921875,0.25187787828947367,0.0,0.0,0.0,166.67154947916666,288.48673046875,1152000,1280000,2432000 diff --git a/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/summary.json b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e34889f0630ff34dd866ae75cfaed2dfcc7dade6 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2", + "id_acc": 0.36663975694444445, + "ood_acc": 0.1485921875, + "total_acc": 0.25187787828947367, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 166.67154947916666, + "ood_avg_resp_len": 288.48673046875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 612567, + "answer_accuracy": 0.25187787828947367, + "avg_response_len": 230.78480263157894, + "resp_tokens_sum": 561268640, + "pass_at_k": { + "pass@1": 0.25187787828947367, + "pass@2": 0.2898682203170242, + "pass@4": 0.3256227865135233, + "pass@8": 0.36177042915407004, + "pass@16": 0.39986189207334566, + "pass@32": 0.4401650877899612, + "pass@64": 0.48260372393310047, + "pass@128": 0.5264210526315789 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.185359375, + "pass@2": 0.22665095964566928, + "pass@4": 0.26727793813273304, + "pass@8": 0.30954397222044167, + "pass@16": 0.35545953862561114, + "pass@32": 0.406521544874947, + "pass@64": 0.46236825402274806, + "pass@128": 0.518 + }, + "2": { + "pass@1": 0.9783828125, + "pass@2": 0.985677288385827, + "pass@4": 0.9900933495500566, + "pass@8": 0.9934069664363652, + "pass@16": 0.9959995639693671, + "pass@32": 0.9980012422309414, + "pass@64": 0.9993614077206364, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.8024375, + "pass@2": 0.8637383120078743, + "pass@4": 0.9026073085864265, + "pass@8": 0.9289196725664263, + "pass@16": 0.9472079819152873, + "pass@32": 0.9598822779486047, + "pass@64": 0.9685287078858967, + "pass@128": 0.974 + }, + "4": { + "pass@1": 0.2260703125, + "pass@2": 0.25332062007874023, + "pass@4": 0.28066603299587534, + "pass@8": 0.3103555783316386, + "pass@16": 0.34315392227388125, + "pass@32": 0.3783126435894053, + "pass@64": 0.41367619377252585, + "pass@128": 0.448 + }, + "5": { + "pass@1": 0.2620703125, + "pass@2": 0.29352362204724425, + "pass@4": 0.3214579548181478, + "pass@8": 0.3493021403707522, + "pass@16": 0.3807327379659776, + "pass@32": 0.4177264064718202, + "pass@64": 0.4594949425831975, + "pass@128": 0.501 + }, + "6": { + "pass@1": 0.257390625, + "pass@2": 0.2899992618110238, + "pass@4": 0.3218167166291711, + "pass@8": 0.3592720826099001, + "pass@16": 0.4044761271292741, + "pass@32": 0.456011217362287, + "pass@64": 0.5093909478948577, + "pass@128": 0.557 + }, + "7": { + "pass@1": 0.2025703125, + "pass@2": 0.2358702017716536, + "pass@4": 0.27075396878515157, + "pass@8": 0.3118831160712313, + "pass@16": 0.36057097564467266, + "pass@32": 0.41305512983021886, + "pass@64": 0.4655551878213001, + "pass@128": 0.516 + }, + "8": { + "pass@1": 0.200203125, + "pass@2": 0.23747268700787386, + "pass@4": 0.27527332133483295, + "pass@8": 0.3189794590222602, + "pass@16": 0.37092671699208357, + "pass@32": 0.4298585350973072, + "pass@64": 0.4941520825565573, + "pass@128": 0.566 + }, + "9": { + "pass@1": 0.1852734375, + "pass@2": 0.22361072834645673, + "pass@4": 0.26136434420697385, + "pass@8": 0.30341340770066794, + "pass@16": 0.35208361376766595, + "pass@32": 0.40723850445797893, + "pass@64": 0.4669966102924466, + "pass@128": 0.528 + }, + "11": { + "pass@1": 0.181328125, + "pass@2": 0.22308956692913356, + "pass@4": 0.2620878564866889, + "pass@8": 0.30073618152874915, + "pass@16": 0.34229874395712756, + "pass@32": 0.38838180923282895, + "pass@64": 0.4374700679217919, + "pass@128": 0.487 + }, + "12": { + "pass@1": 0.1834296875, + "pass@2": 0.22520152559055115, + "pass@4": 0.2631668171166101, + "pass@8": 0.2993326883550847, + "pass@16": 0.3351238797250115, + "pass@32": 0.37152423224352726, + "pass@64": 0.4098912760329702, + "pass@128": 0.451 + }, + "13": { + "pass@1": 0.149203125, + "pass@2": 0.1888870570866141, + "pass@4": 0.2266780743344578, + "pass@8": 0.26277767341577507, + "pass@16": 0.29816678392720575, + "pass@32": 0.335296891994066, + "pass@64": 0.3763319045836284, + "pass@128": 0.422 + }, + "14": { + "pass@1": 0.1473671875, + "pass@2": 0.19373252952755887, + "pass@4": 0.2384486215785524, + "pass@8": 0.28300815279693214, + "pass@16": 0.32785614148323466, + "pass@32": 0.37277487753458227, + "pass@64": 0.4184688571757885, + "pass@128": 0.466 + }, + "15": { + "pass@1": 0.1339375, + "pass@2": 0.17233956692913383, + "pass@4": 0.21110731636670393, + "pass@8": 0.2508732570464338, + "pass@16": 0.29181007808554676, + "pass@32": 0.3342102860408758, + "pass@64": 0.3805825749395631, + "pass@128": 0.43 + }, + "16": { + "pass@1": 0.1640390625, + "pass@2": 0.20756471456692913, + "pass@4": 0.2486257512185974, + "pass@8": 0.28807563999679026, + "pass@16": 0.32744718667297784, + "pass@32": 0.36867748793616795, + "pass@64": 0.41376173730866433, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.1465625, + "pass@2": 0.19127854330708668, + "pass@4": 0.23576637992125982, + "pass@8": 0.27938039291798156, + "pass@16": 0.3211327072075191, + "pass@32": 0.3608730422253764, + "pass@64": 0.40151002182685186, + "pass@128": 0.443 + }, + "18": { + "pass@1": 0.1308125, + "pass@2": 0.17329847440944876, + "pass@4": 0.2158036766966628, + "pass@8": 0.2579953436262119, + "pass@16": 0.3009428053842477, + "pass@32": 0.3455988365948172, + "pass@64": 0.3931462119418665, + "pass@128": 0.445 + }, + "19": { + "pass@1": 0.1276875, + "pass@2": 0.1649736712598425, + "pass@4": 0.20048644581927239, + "pass@8": 0.23576637534301864, + "pass@16": 0.2727791010100153, + "pass@32": 0.31083187841575016, + "pass@64": 0.34949288202114465, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.1215546875, + "pass@2": 0.15726685531496062, + "pass@4": 0.1933510691788525, + "pass@8": 0.23061605357061016, + "pass@16": 0.26920734365662685, + "pass@32": 0.3083598239277578, + "pass@64": 0.3492908864263997, + "pass@128": 0.397 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1edb77e95c6363a6c050b955a3f43686160cb300 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2811d042f9c03f54da03971ae52ed5faa25446a48ef1a863f847369f6d6d073 +size 1677016444 diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_metrics.json b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..439584e8203af32ec5385554634e14d71de2ed40 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2", + "id": { + "answer_accuracy": 0.9839444444444444, + "avg_response_len": 179.58795572916668, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9839444444444444, + "pass@2": 0.9910161444663111, + "pass@4": 0.9943455732200136, + "pass@8": 0.9963765232476208, + "pass@16": 0.997662089913169, + "pass@32": 0.9985092691769479, + "pass@64": 0.9990977855012593, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9546875, + "pass@2": 0.9730445374015754, + "pass@4": 0.9828871964754408, + "pass@8": 0.9891018606541251, + "pass@16": 0.9926498150634105, + "pass@32": 0.994932112165362, + "pass@64": 0.9965873894580104, + "pass@128": 0.998 + }, + "2": { + "pass@1": 0.9998984375, + "pass@2": 0.9999982775590551, + "pass@4": 0.9999999998125234, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9883828125, + "pass@2": 0.9940301427165356, + "pass@4": 0.9966966247656541, + "pass@8": 0.99820105519777, + "pass@16": 0.9990410767030942, + "pass@32": 0.9994276601048258, + "pass@64": 0.9997519563675498, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.993015625, + "pass@2": 0.9953889025590553, + "pass@4": 0.9964886990063742, + "pass@8": 0.9973319427137337, + "pass@16": 0.9981664031982023, + "pass@32": 0.9988859425272113, + "pass@64": 0.9994401641002045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9886875, + "pass@2": 0.9924042814960632, + "pass@4": 0.9947509217285339, + "pass@8": 0.9967725128078163, + "pass@16": 0.9983531259664956, + "pass@32": 0.9992857895097825, + "pass@64": 0.99974859315459, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9875234375, + "pass@2": 0.9925851377952758, + "pass@4": 0.9950138434570678, + "pass@8": 0.9963189326739829, + "pass@16": 0.9969774564839137, + "pass@32": 0.9974501645834012, + "pass@64": 0.9979993322645275, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.98353125, + "pass@2": 0.9901850393700791, + "pass@4": 0.9934791892575929, + "pass@8": 0.9957244280575469, + "pass@16": 0.9972930086824286, + "pass@32": 0.9982314142734684, + "pass@64": 0.9987439452641191, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.985734375, + "pass@2": 0.9950024606299215, + "pass@4": 0.9978733741094861, + "pass@8": 0.9989648417906454, + "pass@16": 0.99951108430717, + "pass@32": 0.9998281678821137, + "pass@64": 0.9999861762742875, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9740390625, + "pass@2": 0.9865065206692918, + "pass@4": 0.991920310367454, + "pass@8": 0.9949731353329699, + "pass@16": 0.9969668388138072, + "pass@32": 0.9985421715463704, + "pass@64": 0.9996225126280432, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9546875, + "2": 0.9998984375, + "3": 0.9883828125, + "4": 0.993015625, + "5": 0.9886875, + "6": 0.9875234375, + "7": 0.98353125, + "8": 0.985734375, + "9": 0.9740390625 + }, + "per_op_avg_response_len": { + "10": 250.523359375, + "2": 104.6227578125, + "3": 124.8024765625, + "4": 143.795578125, + "5": 167.9442109375, + "6": 178.281171875, + "7": 193.9256640625, + "8": 217.5994921875, + "9": 234.796890625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 250.523359375, + "2": 104.6227578125, + "3": 124.8024765625, + "4": 143.795578125, + "5": 167.9442109375, + "6": 178.281171875, + "7": 193.9256640625, + "8": 217.5994921875, + "9": 234.796890625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 1133504, + "answer_accuracy": 0.9839444444444444, + "avg_response_len": 179.58795572916668, + "resp_tokens_sum": 206885325, + "pass_at_k": { + "pass@1": 0.9839444444444444, + "pass@2": 0.9910161444663111, + "pass@4": 0.9943455732200136, + "pass@8": 0.9963765232476208, + "pass@16": 0.997662089913169, + "pass@32": 0.9985092691769479, + "pass@64": 0.9990977855012593, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9546875, + "pass@2": 0.9730445374015754, + "pass@4": 0.9828871964754408, + "pass@8": 0.9891018606541251, + "pass@16": 0.9926498150634105, + "pass@32": 0.994932112165362, + "pass@64": 0.9965873894580104, + "pass@128": 0.998 + }, + "2": { + "pass@1": 0.9998984375, + "pass@2": 0.9999982775590551, + "pass@4": 0.9999999998125234, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9883828125, + "pass@2": 0.9940301427165356, + "pass@4": 0.9966966247656541, + "pass@8": 0.99820105519777, + "pass@16": 0.9990410767030942, + "pass@32": 0.9994276601048258, + "pass@64": 0.9997519563675498, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.993015625, + "pass@2": 0.9953889025590553, + "pass@4": 0.9964886990063742, + "pass@8": 0.9973319427137337, + "pass@16": 0.9981664031982023, + "pass@32": 0.9988859425272113, + "pass@64": 0.9994401641002045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9886875, + "pass@2": 0.9924042814960632, + "pass@4": 0.9947509217285339, + "pass@8": 0.9967725128078163, + "pass@16": 0.9983531259664956, + "pass@32": 0.9992857895097825, + "pass@64": 0.99974859315459, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9875234375, + "pass@2": 0.9925851377952758, + "pass@4": 0.9950138434570678, + "pass@8": 0.9963189326739829, + "pass@16": 0.9969774564839137, + "pass@32": 0.9974501645834012, + "pass@64": 0.9979993322645275, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.98353125, + "pass@2": 0.9901850393700791, + "pass@4": 0.9934791892575929, + "pass@8": 0.9957244280575469, + "pass@16": 0.9972930086824286, + "pass@32": 0.9982314142734684, + "pass@64": 0.9987439452641191, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.985734375, + "pass@2": 0.9950024606299215, + "pass@4": 0.9978733741094861, + "pass@8": 0.9989648417906454, + "pass@16": 0.99951108430717, + "pass@32": 0.9998281678821137, + "pass@64": 0.9999861762742875, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9740390625, + "pass@2": 0.9865065206692918, + "pass@4": 0.991920310367454, + "pass@8": 0.9949731353329699, + "pass@16": 0.9969668388138072, + "pass@32": 0.9985421715463704, + "pass@64": 0.9996225126280432, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.87363515625, + "avg_response_len": 344.30240625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.87363515625, + "pass@2": 0.9241072957676776, + "pass@4": 0.9497318981908452, + "pass@8": 0.9653800207485672, + "pass@16": 0.9760106534543187, + "pass@32": 0.9834210784812789, + "pass@64": 0.9886265415906693, + "pass@128": 0.9921 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.934125, + "pass@2": 0.9581823326771663, + "pass@4": 0.9699951094863142, + "pass@8": 0.9779864505029263, + "pass@16": 0.9840287648870543, + "pass@32": 0.9881931976864097, + "pass@64": 0.9908105195020979, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.9199375, + "pass@2": 0.944433562992127, + "pass@4": 0.9581206651668547, + "pass@8": 0.9682996266573928, + "pass@16": 0.9770597071602081, + "pass@32": 0.9841711541318413, + "pass@64": 0.9890154503221729, + "pass@128": 0.992 + }, + "13": { + "pass@1": 0.897203125, + "pass@2": 0.9290899360236233, + "pass@4": 0.9493992965879263, + "pass@8": 0.9646426969861244, + "pass@16": 0.9757579562792116, + "pass@32": 0.9833918299978378, + "pass@64": 0.9888488618758717, + "pass@128": 0.993 + }, + "14": { + "pass@1": 0.899921875, + "pass@2": 0.9371103592519698, + "pass@4": 0.9551973012748411, + "pass@8": 0.9657808824969042, + "pass@16": 0.973052534504651, + "pass@32": 0.9788486676562278, + "pass@64": 0.9837226062715202, + "pass@128": 0.988 + }, + "15": { + "pass@1": 0.89325, + "pass@2": 0.9371788877952778, + "pass@4": 0.9602339167604054, + "pass@8": 0.9756663620660364, + "pass@16": 0.9856376489789871, + "pass@32": 0.9911178498591497, + "pass@64": 0.993851749241221, + "pass@128": 0.995 + }, + "16": { + "pass@1": 0.8801484375, + "pass@2": 0.9294427903543326, + "pass@4": 0.9535912003187091, + "pass@8": 0.9686415347892804, + "pass@16": 0.9791797640259295, + "pass@32": 0.9864128226208031, + "pass@64": 0.9911713447769573, + "pass@128": 0.994 + }, + "17": { + "pass@1": 0.880625, + "pass@2": 0.9352626722440971, + "pass@4": 0.9590243515185597, + "pass@8": 0.9719596644227715, + "pass@16": 0.9803682728482807, + "pass@32": 0.9863817099303188, + "pass@64": 0.9907030642309558, + "pass@128": 0.994 + }, + "18": { + "pass@1": 0.83946875, + "pass@2": 0.9072063238189, + "pass@4": 0.9399341278590165, + "pass@8": 0.9584937825373301, + "pass@16": 0.9705764209880989, + "pass@32": 0.9792101734137804, + "pass@64": 0.9853420336423154, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.813203125, + "pass@2": 0.8955441683070875, + "pass@4": 0.9368056940382458, + "pass@8": 0.9597209780216617, + "pass@16": 0.9737491833403223, + "pass@32": 0.9829418766592732, + "pass@64": 0.989735662698387, + "pass@128": 0.995 + }, + "20": { + "pass@1": 0.77846875, + "pass@2": 0.8676219242125998, + "pass@4": 0.9150173188976384, + "pass@8": 0.9426082290052561, + "pass@16": 0.9606962815304314, + "pass@32": 0.9735415028571198, + "pass@64": 0.9830641233451889, + "pass@128": 0.989 + } + }, + "per_op_accuracy": { + "11": 0.934125, + "12": 0.9199375, + "13": 0.897203125, + "14": 0.899921875, + "15": 0.89325, + "16": 0.8801484375, + "17": 0.880625, + "18": 0.83946875, + "19": 0.813203125, + "20": 0.77846875 + }, + "per_op_avg_response_len": { + "11": 270.862703125, + "12": 282.4895234375, + "13": 292.799125, + "14": 321.3767109375, + "15": 336.694265625, + "16": 362.2503125, + "17": 373.79278125, + "18": 387.667828125, + "19": 399.22096875, + "20": 415.86984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 270.862703125, + "12": 282.4895234375, + "13": 292.799125, + "14": 321.3767109375, + "15": 336.694265625, + "16": 362.2503125, + "17": 373.79278125, + "18": 387.667828125, + "19": 399.22096875, + "20": 415.86984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 1118253, + "answer_accuracy": 0.87363515625, + "avg_response_len": 344.30240625, + "resp_tokens_sum": 440707080, + "pass_at_k": { + "pass@1": 0.87363515625, + "pass@2": 0.9241072957676776, + "pass@4": 0.9497318981908452, + "pass@8": 0.9653800207485672, + "pass@16": 0.9760106534543187, + "pass@32": 0.9834210784812789, + "pass@64": 0.9886265415906693, + "pass@128": 0.9921 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.934125, + "pass@2": 0.9581823326771663, + "pass@4": 0.9699951094863142, + "pass@8": 0.9779864505029263, + "pass@16": 0.9840287648870543, + "pass@32": 0.9881931976864097, + "pass@64": 0.9908105195020979, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.9199375, + "pass@2": 0.944433562992127, + "pass@4": 0.9581206651668547, + "pass@8": 0.9682996266573928, + "pass@16": 0.9770597071602081, + "pass@32": 0.9841711541318413, + "pass@64": 0.9890154503221729, + "pass@128": 0.992 + }, + "13": { + "pass@1": 0.897203125, + "pass@2": 0.9290899360236233, + "pass@4": 0.9493992965879263, + "pass@8": 0.9646426969861244, + "pass@16": 0.9757579562792116, + "pass@32": 0.9833918299978378, + "pass@64": 0.9888488618758717, + "pass@128": 0.993 + }, + "14": { + "pass@1": 0.899921875, + "pass@2": 0.9371103592519698, + "pass@4": 0.9551973012748411, + "pass@8": 0.9657808824969042, + "pass@16": 0.973052534504651, + "pass@32": 0.9788486676562278, + "pass@64": 0.9837226062715202, + "pass@128": 0.988 + }, + "15": { + "pass@1": 0.89325, + "pass@2": 0.9371788877952778, + "pass@4": 0.9602339167604054, + "pass@8": 0.9756663620660364, + "pass@16": 0.9856376489789871, + "pass@32": 0.9911178498591497, + "pass@64": 0.993851749241221, + "pass@128": 0.995 + }, + "16": { + "pass@1": 0.8801484375, + "pass@2": 0.9294427903543326, + "pass@4": 0.9535912003187091, + "pass@8": 0.9686415347892804, + "pass@16": 0.9791797640259295, + "pass@32": 0.9864128226208031, + "pass@64": 0.9911713447769573, + "pass@128": 0.994 + }, + "17": { + "pass@1": 0.880625, + "pass@2": 0.9352626722440971, + "pass@4": 0.9590243515185597, + "pass@8": 0.9719596644227715, + "pass@16": 0.9803682728482807, + "pass@32": 0.9863817099303188, + "pass@64": 0.9907030642309558, + "pass@128": 0.994 + }, + "18": { + "pass@1": 0.83946875, + "pass@2": 0.9072063238189, + "pass@4": 0.9399341278590165, + "pass@8": 0.9584937825373301, + "pass@16": 0.9705764209880989, + "pass@32": 0.9792101734137804, + "pass@64": 0.9853420336423154, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.813203125, + "pass@2": 0.8955441683070875, + "pass@4": 0.9368056940382458, + "pass@8": 0.9597209780216617, + "pass@16": 0.9737491833403223, + "pass@32": 0.9829418766592732, + "pass@64": 0.989735662698387, + "pass@128": 0.995 + }, + "20": { + "pass@1": 0.77846875, + "pass@2": 0.8676219242125998, + "pass@4": 0.9150173188976384, + "pass@8": 0.9426082290052561, + "pass@16": 0.9606962815304314, + "pass@32": 0.9735415028571198, + "pass@64": 0.9830641233451889, + "pass@128": 0.989 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.9258869243421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9258869243421053, + "pass@2": 0.9558009609407194, + "pass@4": 0.9708646916257092, + "pass@8": 0.9800625745639231, + "pass@16": 0.9862665970400866, + "pass@32": 0.9905681161792315, + "pass@64": 0.993586604495685, + "pass@128": 0.9956315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9546875, + "pass@2": 0.9730445374015754, + "pass@4": 0.9828871964754408, + "pass@8": 0.9891018606541251, + "pass@16": 0.9926498150634105, + "pass@32": 0.994932112165362, + "pass@64": 0.9965873894580104, + "pass@128": 0.998 + }, + "2": { + "pass@1": 0.9998984375, + "pass@2": 0.9999982775590551, + "pass@4": 0.9999999998125234, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9883828125, + "pass@2": 0.9940301427165356, + "pass@4": 0.9966966247656541, + "pass@8": 0.99820105519777, + "pass@16": 0.9990410767030942, + "pass@32": 0.9994276601048258, + "pass@64": 0.9997519563675498, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.993015625, + "pass@2": 0.9953889025590553, + "pass@4": 0.9964886990063742, + "pass@8": 0.9973319427137337, + "pass@16": 0.9981664031982023, + "pass@32": 0.9988859425272113, + "pass@64": 0.9994401641002045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9886875, + "pass@2": 0.9924042814960632, + "pass@4": 0.9947509217285339, + "pass@8": 0.9967725128078163, + "pass@16": 0.9983531259664956, + "pass@32": 0.9992857895097825, + "pass@64": 0.99974859315459, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9875234375, + "pass@2": 0.9925851377952758, + "pass@4": 0.9950138434570678, + "pass@8": 0.9963189326739829, + "pass@16": 0.9969774564839137, + "pass@32": 0.9974501645834012, + "pass@64": 0.9979993322645275, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.98353125, + "pass@2": 0.9901850393700791, + "pass@4": 0.9934791892575929, + "pass@8": 0.9957244280575469, + "pass@16": 0.9972930086824286, + "pass@32": 0.9982314142734684, + "pass@64": 0.9987439452641191, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.985734375, + "pass@2": 0.9950024606299215, + "pass@4": 0.9978733741094861, + "pass@8": 0.9989648417906454, + "pass@16": 0.99951108430717, + "pass@32": 0.9998281678821137, + "pass@64": 0.9999861762742875, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9740390625, + "pass@2": 0.9865065206692918, + "pass@4": 0.991920310367454, + "pass@8": 0.9949731353329699, + "pass@16": 0.9969668388138072, + "pass@32": 0.9985421715463704, + "pass@64": 0.9996225126280432, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.934125, + "pass@2": 0.9581823326771663, + "pass@4": 0.9699951094863142, + "pass@8": 0.9779864505029263, + "pass@16": 0.9840287648870543, + "pass@32": 0.9881931976864097, + "pass@64": 0.9908105195020979, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.9199375, + "pass@2": 0.944433562992127, + "pass@4": 0.9581206651668547, + "pass@8": 0.9682996266573928, + "pass@16": 0.9770597071602081, + "pass@32": 0.9841711541318413, + "pass@64": 0.9890154503221729, + "pass@128": 0.992 + }, + "13": { + "pass@1": 0.897203125, + "pass@2": 0.9290899360236233, + "pass@4": 0.9493992965879263, + "pass@8": 0.9646426969861244, + "pass@16": 0.9757579562792116, + "pass@32": 0.9833918299978378, + "pass@64": 0.9888488618758717, + "pass@128": 0.993 + }, + "14": { + "pass@1": 0.899921875, + "pass@2": 0.9371103592519698, + "pass@4": 0.9551973012748411, + "pass@8": 0.9657808824969042, + "pass@16": 0.973052534504651, + "pass@32": 0.9788486676562278, + "pass@64": 0.9837226062715202, + "pass@128": 0.988 + }, + "15": { + "pass@1": 0.89325, + "pass@2": 0.9371788877952778, + "pass@4": 0.9602339167604054, + "pass@8": 0.9756663620660364, + "pass@16": 0.9856376489789871, + "pass@32": 0.9911178498591497, + "pass@64": 0.993851749241221, + "pass@128": 0.995 + }, + "16": { + "pass@1": 0.8801484375, + "pass@2": 0.9294427903543326, + "pass@4": 0.9535912003187091, + "pass@8": 0.9686415347892804, + "pass@16": 0.9791797640259295, + "pass@32": 0.9864128226208031, + "pass@64": 0.9911713447769573, + "pass@128": 0.994 + }, + "17": { + "pass@1": 0.880625, + "pass@2": 0.9352626722440971, + "pass@4": 0.9590243515185597, + "pass@8": 0.9719596644227715, + "pass@16": 0.9803682728482807, + "pass@32": 0.9863817099303188, + "pass@64": 0.9907030642309558, + "pass@128": 0.994 + }, + "18": { + "pass@1": 0.83946875, + "pass@2": 0.9072063238189, + "pass@4": 0.9399341278590165, + "pass@8": 0.9584937825373301, + "pass@16": 0.9705764209880989, + "pass@32": 0.9792101734137804, + "pass@64": 0.9853420336423154, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.813203125, + "pass@2": 0.8955441683070875, + "pass@4": 0.9368056940382458, + "pass@8": 0.9597209780216617, + "pass@16": 0.9737491833403223, + "pass@32": 0.9829418766592732, + "pass@64": 0.989735662698387, + "pass@128": 0.995 + }, + "20": { + "pass@1": 0.77846875, + "pass@2": 0.8676219242125998, + "pass@4": 0.9150173188976384, + "pass@8": 0.9426082290052561, + "pass@16": 0.9606962815304314, + "pass@32": 0.9735415028571198, + "pass@64": 0.9830641233451889, + "pass@128": 0.989 + } + }, + "per_op_accuracy": { + "10": 0.9546875, + "2": 0.9998984375, + "3": 0.9883828125, + "4": 0.993015625, + "5": 0.9886875, + "6": 0.9875234375, + "7": 0.98353125, + "8": 0.985734375, + "9": 0.9740390625, + "11": 0.934125, + "12": 0.9199375, + "13": 0.897203125, + "14": 0.899921875, + "15": 0.89325, + "16": 0.8801484375, + "17": 0.880625, + "18": 0.83946875, + "19": 0.813203125, + "20": 0.77846875 + }, + "per_op_avg_response_len": { + "10": 250.523359375, + "2": 104.6227578125, + "3": 124.8024765625, + "4": 143.795578125, + "5": 167.9442109375, + "6": 178.281171875, + "7": 193.9256640625, + "8": 217.5994921875, + "9": 234.796890625, + "11": 270.862703125, + "12": 282.4895234375, + "13": 292.799125, + "14": 321.3767109375, + "15": 336.694265625, + "16": 362.2503125, + "17": 373.79278125, + "18": 387.667828125, + "19": 399.22096875, + "20": 415.86984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 250.523359375, + "2": 104.6227578125, + "3": 124.8024765625, + "4": 143.795578125, + "5": 167.9442109375, + "6": 178.281171875, + "7": 193.9256640625, + "8": 217.5994921875, + "9": 234.796890625, + "11": 270.862703125, + "12": 282.4895234375, + "13": 292.799125, + "14": 321.3767109375, + "15": 336.694265625, + "16": 362.2503125, + "17": 373.79278125, + "18": 387.667828125, + "19": 399.22096875, + "20": 415.86984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2251757, + "answer_accuracy": 0.9258869243421053, + "avg_response_len": 266.27977179276314, + "resp_tokens_sum": 647592405, + "pass_at_k": { + "pass@1": 0.9258869243421053, + "pass@2": 0.9558009609407194, + "pass@4": 0.9708646916257092, + "pass@8": 0.9800625745639231, + "pass@16": 0.9862665970400866, + "pass@32": 0.9905681161792315, + "pass@64": 0.993586604495685, + "pass@128": 0.9956315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9546875, + "pass@2": 0.9730445374015754, + "pass@4": 0.9828871964754408, + "pass@8": 0.9891018606541251, + "pass@16": 0.9926498150634105, + "pass@32": 0.994932112165362, + "pass@64": 0.9965873894580104, + "pass@128": 0.998 + }, + "2": { + "pass@1": 0.9998984375, + "pass@2": 0.9999982775590551, + "pass@4": 0.9999999998125234, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9883828125, + "pass@2": 0.9940301427165356, + "pass@4": 0.9966966247656541, + "pass@8": 0.99820105519777, + "pass@16": 0.9990410767030942, + "pass@32": 0.9994276601048258, + "pass@64": 0.9997519563675498, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.993015625, + "pass@2": 0.9953889025590553, + "pass@4": 0.9964886990063742, + "pass@8": 0.9973319427137337, + "pass@16": 0.9981664031982023, + "pass@32": 0.9988859425272113, + "pass@64": 0.9994401641002045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9886875, + "pass@2": 0.9924042814960632, + "pass@4": 0.9947509217285339, + "pass@8": 0.9967725128078163, + "pass@16": 0.9983531259664956, + "pass@32": 0.9992857895097825, + "pass@64": 0.99974859315459, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9875234375, + "pass@2": 0.9925851377952758, + "pass@4": 0.9950138434570678, + "pass@8": 0.9963189326739829, + "pass@16": 0.9969774564839137, + "pass@32": 0.9974501645834012, + "pass@64": 0.9979993322645275, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.98353125, + "pass@2": 0.9901850393700791, + "pass@4": 0.9934791892575929, + "pass@8": 0.9957244280575469, + "pass@16": 0.9972930086824286, + "pass@32": 0.9982314142734684, + "pass@64": 0.9987439452641191, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.985734375, + "pass@2": 0.9950024606299215, + "pass@4": 0.9978733741094861, + "pass@8": 0.9989648417906454, + "pass@16": 0.99951108430717, + "pass@32": 0.9998281678821137, + "pass@64": 0.9999861762742875, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9740390625, + "pass@2": 0.9865065206692918, + "pass@4": 0.991920310367454, + "pass@8": 0.9949731353329699, + "pass@16": 0.9969668388138072, + "pass@32": 0.9985421715463704, + "pass@64": 0.9996225126280432, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.934125, + "pass@2": 0.9581823326771663, + "pass@4": 0.9699951094863142, + "pass@8": 0.9779864505029263, + "pass@16": 0.9840287648870543, + "pass@32": 0.9881931976864097, + "pass@64": 0.9908105195020979, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.9199375, + "pass@2": 0.944433562992127, + "pass@4": 0.9581206651668547, + "pass@8": 0.9682996266573928, + "pass@16": 0.9770597071602081, + "pass@32": 0.9841711541318413, + "pass@64": 0.9890154503221729, + "pass@128": 0.992 + }, + "13": { + "pass@1": 0.897203125, + "pass@2": 0.9290899360236233, + "pass@4": 0.9493992965879263, + "pass@8": 0.9646426969861244, + "pass@16": 0.9757579562792116, + "pass@32": 0.9833918299978378, + "pass@64": 0.9888488618758717, + "pass@128": 0.993 + }, + "14": { + "pass@1": 0.899921875, + "pass@2": 0.9371103592519698, + "pass@4": 0.9551973012748411, + "pass@8": 0.9657808824969042, + "pass@16": 0.973052534504651, + "pass@32": 0.9788486676562278, + "pass@64": 0.9837226062715202, + "pass@128": 0.988 + }, + "15": { + "pass@1": 0.89325, + "pass@2": 0.9371788877952778, + "pass@4": 0.9602339167604054, + "pass@8": 0.9756663620660364, + "pass@16": 0.9856376489789871, + "pass@32": 0.9911178498591497, + "pass@64": 0.993851749241221, + "pass@128": 0.995 + }, + "16": { + "pass@1": 0.8801484375, + "pass@2": 0.9294427903543326, + "pass@4": 0.9535912003187091, + "pass@8": 0.9686415347892804, + "pass@16": 0.9791797640259295, + "pass@32": 0.9864128226208031, + "pass@64": 0.9911713447769573, + "pass@128": 0.994 + }, + "17": { + "pass@1": 0.880625, + "pass@2": 0.9352626722440971, + "pass@4": 0.9590243515185597, + "pass@8": 0.9719596644227715, + "pass@16": 0.9803682728482807, + "pass@32": 0.9863817099303188, + "pass@64": 0.9907030642309558, + "pass@128": 0.994 + }, + "18": { + "pass@1": 0.83946875, + "pass@2": 0.9072063238189, + "pass@4": 0.9399341278590165, + "pass@8": 0.9584937825373301, + "pass@16": 0.9705764209880989, + "pass@32": 0.9792101734137804, + "pass@64": 0.9853420336423154, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.813203125, + "pass@2": 0.8955441683070875, + "pass@4": 0.9368056940382458, + "pass@8": 0.9597209780216617, + "pass@16": 0.9737491833403223, + "pass@32": 0.9829418766592732, + "pass@64": 0.989735662698387, + "pass@128": 0.995 + }, + "20": { + "pass@1": 0.77846875, + "pass@2": 0.8676219242125998, + "pass@4": 0.9150173188976384, + "pass@8": 0.9426082290052561, + "pass@16": 0.9606962815304314, + "pass@32": 0.9735415028571198, + "pass@64": 0.9830641233451889, + "pass@128": 0.989 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..93b693299cfb041972bb69240b9df05336e25a63 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:793a4fe52fb17df2faecf2a5ef5a7eabda09b9e6d9b36d18aa80a0ee94a0eb4f +size 2616437443 diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl.graph_results.jsonl b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c28143188c4f46334c56b2db828f62304a07aff6 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09429545527c271993efc0d1a8e1635d93d57b3f37af0797815b57fff228ebe8 +size 374967939 diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl.graph_results.jsonl b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7ee6395601a76722c66cafa7e8b68c401b84ec78 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8a878c91761604173a2ca552c893b90cc47652a29eb115346f75f7771daa9e1 +size 423255040 diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9d3c5bda4a3ef9c739563f5c9cb8a5fdf04ff415 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/graphs", "generated_path": "results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2", "results_dir": "results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs", "per_file": [{"file": "results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 25511, "value_mismatch": 7776, "dependency_mismatch": 1046981, "answer_mismatch": 18496, "extra_nodes": 1126479, "missing_nodes": 1126489, "perfect_with_extra": 0, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "dependency_mismatch": 1046981, "extra_nodes": 1126479, "missing_nodes": 1126489, "answer_mismatch": 18496, "value_mismatch": 7776, "perfect": 25511}}, "results_path": "results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 125510, "dependency_mismatch": 1259672, "answer_mismatch": 161747, "extra_nodes": 1279539, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 125510, "dependency_mismatch": 1259672, "answer_mismatch": 161747, "extra_nodes": 1279539, "missing_nodes": 1280000}}, "results_path": "results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 25511, "value_mismatch": 133286, "dependency_mismatch": 2306653, "answer_mismatch": 180243, "extra_nodes": 2406018, "missing_nodes": 2406489, "perfect_with_extra": 0, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "dependency_mismatch": 2306653, "extra_nodes": 2406018, "missing_nodes": 2406489, "answer_mismatch": 180243, "value_mismatch": 133286, "perfect": 25511}}}} diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/results.json b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..03635b3dff73784692aa8cc4840da132e7882b20 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.188, + "pass@2": 0.239, + "pass@4": 0.279, + "pass@8": 0.316, + "pass@16": 0.346, + "pass@32": 0.382, + "pass@64": 0.414, + "pass@128": 0.452 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.188, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.239, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.279, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.316, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.346, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.382, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.414, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.452, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.020888888888888887, + "pass@2": 0.026555555555555554, + "pass@4": 0.031, + "pass@8": 0.035111111111111114, + "pass@16": 0.03844444444444445, + "pass@32": 0.042444444444444444, + "pass@64": 0.046, + "pass@128": 0.050222222222222224 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.188, + "pass@2": 0.239, + "pass@4": 0.279, + "pass@8": 0.316, + "pass@16": 0.346, + "pass@32": 0.382, + "pass@64": 0.414, + "pass@128": 0.452 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.188, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.239, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.279, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.316, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.346, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.382, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.414, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.452, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.020888888888888887, + "pass@2": 0.026555555555555554, + "pass@4": 0.031, + "pass@8": 0.035111111111111114, + "pass@16": 0.03844444444444445, + "pass@32": 0.042444444444444444, + "pass@64": 0.046, + "pass@128": 0.050222222222222224 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/summary.csv b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..aa7ce595d8095492d1374158707674ae310d49cf --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2,0.9839444444444444,0.87363515625,0.9258869243421053,0.0,0.0,0.0,179.58795572916668,344.30240625,1152000,1280000,2432000 diff --git a/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/summary.json b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8df7e300989dc352bbb800061317c012281cf6ce --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2", + "id_acc": 0.9839444444444444, + "ood_acc": 0.87363515625, + "total_acc": 0.9258869243421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 179.58795572916668, + "ood_avg_resp_len": 344.30240625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2251757, + "answer_accuracy": 0.9258869243421053, + "avg_response_len": 266.27977179276314, + "resp_tokens_sum": 647592405, + "pass_at_k": { + "pass@1": 0.9258869243421053, + "pass@2": 0.9558009609407194, + "pass@4": 0.9708646916257092, + "pass@8": 0.9800625745639231, + "pass@16": 0.9862665970400866, + "pass@32": 0.9905681161792315, + "pass@64": 0.993586604495685, + "pass@128": 0.9956315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9546875, + "pass@2": 0.9730445374015754, + "pass@4": 0.9828871964754408, + "pass@8": 0.9891018606541251, + "pass@16": 0.9926498150634105, + "pass@32": 0.994932112165362, + "pass@64": 0.9965873894580104, + "pass@128": 0.998 + }, + "2": { + "pass@1": 0.9998984375, + "pass@2": 0.9999982775590551, + "pass@4": 0.9999999998125234, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9883828125, + "pass@2": 0.9940301427165356, + "pass@4": 0.9966966247656541, + "pass@8": 0.99820105519777, + "pass@16": 0.9990410767030942, + "pass@32": 0.9994276601048258, + "pass@64": 0.9997519563675498, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.993015625, + "pass@2": 0.9953889025590553, + "pass@4": 0.9964886990063742, + "pass@8": 0.9973319427137337, + "pass@16": 0.9981664031982023, + "pass@32": 0.9988859425272113, + "pass@64": 0.9994401641002045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9886875, + "pass@2": 0.9924042814960632, + "pass@4": 0.9947509217285339, + "pass@8": 0.9967725128078163, + "pass@16": 0.9983531259664956, + "pass@32": 0.9992857895097825, + "pass@64": 0.99974859315459, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9875234375, + "pass@2": 0.9925851377952758, + "pass@4": 0.9950138434570678, + "pass@8": 0.9963189326739829, + "pass@16": 0.9969774564839137, + "pass@32": 0.9974501645834012, + "pass@64": 0.9979993322645275, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.98353125, + "pass@2": 0.9901850393700791, + "pass@4": 0.9934791892575929, + "pass@8": 0.9957244280575469, + "pass@16": 0.9972930086824286, + "pass@32": 0.9982314142734684, + "pass@64": 0.9987439452641191, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.985734375, + "pass@2": 0.9950024606299215, + "pass@4": 0.9978733741094861, + "pass@8": 0.9989648417906454, + "pass@16": 0.99951108430717, + "pass@32": 0.9998281678821137, + "pass@64": 0.9999861762742875, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9740390625, + "pass@2": 0.9865065206692918, + "pass@4": 0.991920310367454, + "pass@8": 0.9949731353329699, + "pass@16": 0.9969668388138072, + "pass@32": 0.9985421715463704, + "pass@64": 0.9996225126280432, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.934125, + "pass@2": 0.9581823326771663, + "pass@4": 0.9699951094863142, + "pass@8": 0.9779864505029263, + "pass@16": 0.9840287648870543, + "pass@32": 0.9881931976864097, + "pass@64": 0.9908105195020979, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.9199375, + "pass@2": 0.944433562992127, + "pass@4": 0.9581206651668547, + "pass@8": 0.9682996266573928, + "pass@16": 0.9770597071602081, + "pass@32": 0.9841711541318413, + "pass@64": 0.9890154503221729, + "pass@128": 0.992 + }, + "13": { + "pass@1": 0.897203125, + "pass@2": 0.9290899360236233, + "pass@4": 0.9493992965879263, + "pass@8": 0.9646426969861244, + "pass@16": 0.9757579562792116, + "pass@32": 0.9833918299978378, + "pass@64": 0.9888488618758717, + "pass@128": 0.993 + }, + "14": { + "pass@1": 0.899921875, + "pass@2": 0.9371103592519698, + "pass@4": 0.9551973012748411, + "pass@8": 0.9657808824969042, + "pass@16": 0.973052534504651, + "pass@32": 0.9788486676562278, + "pass@64": 0.9837226062715202, + "pass@128": 0.988 + }, + "15": { + "pass@1": 0.89325, + "pass@2": 0.9371788877952778, + "pass@4": 0.9602339167604054, + "pass@8": 0.9756663620660364, + "pass@16": 0.9856376489789871, + "pass@32": 0.9911178498591497, + "pass@64": 0.993851749241221, + "pass@128": 0.995 + }, + "16": { + "pass@1": 0.8801484375, + "pass@2": 0.9294427903543326, + "pass@4": 0.9535912003187091, + "pass@8": 0.9686415347892804, + "pass@16": 0.9791797640259295, + "pass@32": 0.9864128226208031, + "pass@64": 0.9911713447769573, + "pass@128": 0.994 + }, + "17": { + "pass@1": 0.880625, + "pass@2": 0.9352626722440971, + "pass@4": 0.9590243515185597, + "pass@8": 0.9719596644227715, + "pass@16": 0.9803682728482807, + "pass@32": 0.9863817099303188, + "pass@64": 0.9907030642309558, + "pass@128": 0.994 + }, + "18": { + "pass@1": 0.83946875, + "pass@2": 0.9072063238189, + "pass@4": 0.9399341278590165, + "pass@8": 0.9584937825373301, + "pass@16": 0.9705764209880989, + "pass@32": 0.9792101734137804, + "pass@64": 0.9853420336423154, + "pass@128": 0.989 + }, + "19": { + "pass@1": 0.813203125, + "pass@2": 0.8955441683070875, + "pass@4": 0.9368056940382458, + "pass@8": 0.9597209780216617, + "pass@16": 0.9737491833403223, + "pass@32": 0.9829418766592732, + "pass@64": 0.989735662698387, + "pass@128": 0.995 + }, + "20": { + "pass@1": 0.77846875, + "pass@2": 0.8676219242125998, + "pass@4": 0.9150173188976384, + "pass@8": 0.9426082290052561, + "pass@16": 0.9606962815304314, + "pass@32": 0.9735415028571198, + "pass@64": 0.9830641233451889, + "pass@128": 0.989 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..606280a55c823d7525ee9966140877b2bd358090 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8e6e545086f906f2e1670988967109a061d42b103731267cc7c423d35320e52 +size 1691927385 diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_metrics.json b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..35070648d2c4c27130241a66a05195e32effe61d --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2", + "id": { + "answer_accuracy": 0.970609375, + "avg_response_len": 184.33640538194445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.970609375, + "pass@2": 0.9862094406167796, + "pass@4": 0.9926313236157984, + "pass@8": 0.9957557210757718, + "pass@16": 0.9973273016924448, + "pass@32": 0.9981813363107415, + "pass@64": 0.9986699855766451, + "pass@128": 0.9988888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.913015625, + "pass@2": 0.9545599163385837, + "pass@4": 0.9743599124484438, + "pass@8": 0.9845497950289993, + "pass@16": 0.98992591758449, + "pass@32": 0.9929320417591375, + "pass@64": 0.9947494389701538, + "pass@128": 0.996 + }, + "2": { + "pass@1": 0.99959375, + "pass@2": 0.9999697342519684, + "pass@4": 0.9999997261904762, + "pass@8": 0.9999999999828495, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.989890625, + "pass@2": 0.9958089320866145, + "pass@4": 0.9982959692538436, + "pass@8": 0.9993399787031018, + "pass@16": 0.9998187704659531, + "pass@32": 0.9999808388118459, + "pass@64": 0.9999999379485736, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994609375, + "pass@2": 0.9980883366141732, + "pass@4": 0.9990098940757404, + "pass@8": 0.9992710019844361, + "pass@16": 0.9994928857884559, + "pass@32": 0.9997689651291657, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873828125, + "pass@2": 0.992854822834646, + "pass@4": 0.9956864052305961, + "pass@8": 0.9974935802083053, + "pass@16": 0.9986672794278371, + "pass@32": 0.9994369593888157, + "pass@64": 0.9998741889916245, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9779296875, + "pass@2": 0.9891516978346466, + "pass@4": 0.993640552118485, + "pass@8": 0.9963302804712767, + "pass@16": 0.998015205389486, + "pass@32": 0.9992175807153899, + "pass@64": 0.999897068560936, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9710390625, + "pass@2": 0.987645546259844, + "pass@4": 0.9941635037495307, + "pass@8": 0.9969810934042619, + "pass@16": 0.9978741854718784, + "pass@32": 0.9979971178236134, + "pass@64": 0.9979999996026606, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.9629921875, + "pass@2": 0.9842913385826776, + "pass@4": 0.9905712987439065, + "pass@8": 0.9931610401498477, + "pass@16": 0.9947114227372944, + "pass@32": 0.995840688735689, + "pass@64": 0.9966632229237332, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.93903125, + "pass@2": 0.9735146407480327, + "pass@4": 0.9879546507311577, + "pass@8": 0.9946747197488702, + "pass@16": 0.9974400483666008, + "pass@32": 0.9984578344330189, + "pass@64": 0.9988748320897604, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.913015625, + "2": 0.99959375, + "3": 0.989890625, + "4": 0.994609375, + "5": 0.9873828125, + "6": 0.9779296875, + "7": 0.9710390625, + "8": 0.9629921875, + "9": 0.93903125 + }, + "per_op_avg_response_len": { + "10": 256.4869765625, + "2": 107.1426328125, + "3": 128.4695078125, + "4": 147.98609375, + "5": 170.830171875, + "6": 183.909140625, + "7": 198.936171875, + "8": 223.3921015625, + "9": 241.8748515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 256.4869765625, + "2": 107.1426328125, + "3": 128.4695078125, + "4": 147.98609375, + "5": 170.830171875, + "6": 183.909140625, + "7": 198.936171875, + "8": 223.3921015625, + "9": 241.8748515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 1118142, + "answer_accuracy": 0.970609375, + "avg_response_len": 184.33640538194445, + "resp_tokens_sum": 212355539, + "pass_at_k": { + "pass@1": 0.970609375, + "pass@2": 0.9862094406167796, + "pass@4": 0.9926313236157984, + "pass@8": 0.9957557210757718, + "pass@16": 0.9973273016924448, + "pass@32": 0.9981813363107415, + "pass@64": 0.9986699855766451, + "pass@128": 0.9988888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.913015625, + "pass@2": 0.9545599163385837, + "pass@4": 0.9743599124484438, + "pass@8": 0.9845497950289993, + "pass@16": 0.98992591758449, + "pass@32": 0.9929320417591375, + "pass@64": 0.9947494389701538, + "pass@128": 0.996 + }, + "2": { + "pass@1": 0.99959375, + "pass@2": 0.9999697342519684, + "pass@4": 0.9999997261904762, + "pass@8": 0.9999999999828495, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.989890625, + "pass@2": 0.9958089320866145, + "pass@4": 0.9982959692538436, + "pass@8": 0.9993399787031018, + "pass@16": 0.9998187704659531, + "pass@32": 0.9999808388118459, + "pass@64": 0.9999999379485736, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994609375, + "pass@2": 0.9980883366141732, + "pass@4": 0.9990098940757404, + "pass@8": 0.9992710019844361, + "pass@16": 0.9994928857884559, + "pass@32": 0.9997689651291657, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873828125, + "pass@2": 0.992854822834646, + "pass@4": 0.9956864052305961, + "pass@8": 0.9974935802083053, + "pass@16": 0.9986672794278371, + "pass@32": 0.9994369593888157, + "pass@64": 0.9998741889916245, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9779296875, + "pass@2": 0.9891516978346466, + "pass@4": 0.993640552118485, + "pass@8": 0.9963302804712767, + "pass@16": 0.998015205389486, + "pass@32": 0.9992175807153899, + "pass@64": 0.999897068560936, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9710390625, + "pass@2": 0.987645546259844, + "pass@4": 0.9941635037495307, + "pass@8": 0.9969810934042619, + "pass@16": 0.9978741854718784, + "pass@32": 0.9979971178236134, + "pass@64": 0.9979999996026606, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.9629921875, + "pass@2": 0.9842913385826776, + "pass@4": 0.9905712987439065, + "pass@8": 0.9931610401498477, + "pass@16": 0.9947114227372944, + "pass@32": 0.995840688735689, + "pass@64": 0.9966632229237332, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.93903125, + "pass@2": 0.9735146407480327, + "pass@4": 0.9879546507311577, + "pass@8": 0.9946747197488702, + "pass@16": 0.9974400483666008, + "pass@32": 0.9984578344330189, + "pass@64": 0.9988748320897604, + "pass@128": 0.999 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.75367109375, + "avg_response_len": 352.017434375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.75367109375, + "pass@2": 0.8512200049212475, + "pass@4": 0.9097165535995436, + "pass@8": 0.9443706346400681, + "pass@16": 0.9654192113865045, + "pass@32": 0.9785740674732004, + "pass@64": 0.9869100808088804, + "pass@128": 0.9922 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8736484375, + "pass@2": 0.9301213090551195, + "pass@4": 0.958462992875891, + "pass@8": 0.9746018529563227, + "pass@16": 0.9844361379001465, + "pass@32": 0.9902204108371494, + "pass@64": 0.9933914275171519, + "pass@128": 0.995 + }, + "12": { + "pass@1": 0.865953125, + "pass@2": 0.9204032972440962, + "pass@4": 0.9486816894450689, + "pass@8": 0.9657994139719662, + "pass@16": 0.9770121985407124, + "pass@32": 0.9846340088371232, + "pass@64": 0.9905664884095952, + "pass@128": 0.996 + }, + "13": { + "pass@1": 0.8277890625, + "pass@2": 0.8996813484251981, + "pass@4": 0.938710904293213, + "pass@8": 0.9617088837168377, + "pass@16": 0.9757131222438731, + "pass@32": 0.983740177941991, + "pass@64": 0.9882869729395238, + "pass@128": 0.991 + }, + "14": { + "pass@1": 0.8092734375, + "pass@2": 0.8884940944881912, + "pass@4": 0.9315317574990621, + "pass@8": 0.956514112861527, + "pass@16": 0.973021914723263, + "pass@32": 0.984456875100545, + "pass@64": 0.9919548746245034, + "pass@128": 0.996 + }, + "15": { + "pass@1": 0.7807578125, + "pass@2": 0.8736354576771668, + "pass@4": 0.9264047814960625, + "pass@8": 0.9554910319277518, + "pass@16": 0.9721494395171904, + "pass@32": 0.9830732857749847, + "pass@64": 0.9911087919787471, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.77846875, + "pass@2": 0.8802544291338604, + "pass@4": 0.9362517380014989, + "pass@8": 0.9654782256686752, + "pass@16": 0.9808689703520912, + "pass@32": 0.9898077910135532, + "pass@64": 0.9945683766692341, + "pass@128": 0.996 + }, + "17": { + "pass@1": 0.7288828125, + "pass@2": 0.8466943897637799, + "pass@4": 0.915012037307836, + "pass@8": 0.9521726149843732, + "pass@16": 0.9726557425898653, + "pass@32": 0.9843864841945104, + "pass@64": 0.9921814457320964, + "pass@128": 0.998 + }, + "18": { + "pass@1": 0.6799375, + "pass@2": 0.8053351377952757, + "pass@4": 0.883184441882265, + "pass@8": 0.9293421197147212, + "pass@16": 0.9568947610573432, + "pass@32": 0.973791820757004, + "pass@64": 0.9839928919990625, + "pass@128": 0.991 + }, + "19": { + "pass@1": 0.6220234375, + "pass@2": 0.7577477854330718, + "pass@4": 0.8491333930446202, + "pass@8": 0.9062156892519055, + "pass@16": 0.942281258892309, + "pass@32": 0.9657194038341252, + "pass@64": 0.9804252054829498, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.5699765625, + "pass@2": 0.70983280019685, + "pass@4": 0.8097918001499816, + "pass@8": 0.8763824013466858, + "pass@16": 0.9191585680481961, + "pass@32": 0.9459104164409721, + "pass@64": 0.9626243327359043, + "pass@128": 0.973 + } + }, + "per_op_accuracy": { + "11": 0.8736484375, + "12": 0.865953125, + "13": 0.8277890625, + "14": 0.8092734375, + "15": 0.7807578125, + "16": 0.77846875, + "17": 0.7288828125, + "18": 0.6799375, + "19": 0.6220234375, + "20": 0.5699765625 + }, + "per_op_avg_response_len": { + "11": 277.387671875, + "12": 288.073578125, + "13": 299.479421875, + "14": 328.8324765625, + "15": 344.5523984375, + "16": 371.0299140625, + "17": 382.4528046875, + "18": 396.5237421875, + "19": 407.3039765625, + "20": 424.538359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 277.387671875, + "12": 288.073578125, + "13": 299.479421875, + "14": 328.8324765625, + "15": 344.5523984375, + "16": 371.0299140625, + "17": 382.4528046875, + "18": 396.5237421875, + "19": 407.3039765625, + "20": 424.538359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 964699, + "answer_accuracy": 0.75367109375, + "avg_response_len": 352.017434375, + "resp_tokens_sum": 450582316, + "pass_at_k": { + "pass@1": 0.75367109375, + "pass@2": 0.8512200049212475, + "pass@4": 0.9097165535995436, + "pass@8": 0.9443706346400681, + "pass@16": 0.9654192113865045, + "pass@32": 0.9785740674732004, + "pass@64": 0.9869100808088804, + "pass@128": 0.9922 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8736484375, + "pass@2": 0.9301213090551195, + "pass@4": 0.958462992875891, + "pass@8": 0.9746018529563227, + "pass@16": 0.9844361379001465, + "pass@32": 0.9902204108371494, + "pass@64": 0.9933914275171519, + "pass@128": 0.995 + }, + "12": { + "pass@1": 0.865953125, + "pass@2": 0.9204032972440962, + "pass@4": 0.9486816894450689, + "pass@8": 0.9657994139719662, + "pass@16": 0.9770121985407124, + "pass@32": 0.9846340088371232, + "pass@64": 0.9905664884095952, + "pass@128": 0.996 + }, + "13": { + "pass@1": 0.8277890625, + "pass@2": 0.8996813484251981, + "pass@4": 0.938710904293213, + "pass@8": 0.9617088837168377, + "pass@16": 0.9757131222438731, + "pass@32": 0.983740177941991, + "pass@64": 0.9882869729395238, + "pass@128": 0.991 + }, + "14": { + "pass@1": 0.8092734375, + "pass@2": 0.8884940944881912, + "pass@4": 0.9315317574990621, + "pass@8": 0.956514112861527, + "pass@16": 0.973021914723263, + "pass@32": 0.984456875100545, + "pass@64": 0.9919548746245034, + "pass@128": 0.996 + }, + "15": { + "pass@1": 0.7807578125, + "pass@2": 0.8736354576771668, + "pass@4": 0.9264047814960625, + "pass@8": 0.9554910319277518, + "pass@16": 0.9721494395171904, + "pass@32": 0.9830732857749847, + "pass@64": 0.9911087919787471, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.77846875, + "pass@2": 0.8802544291338604, + "pass@4": 0.9362517380014989, + "pass@8": 0.9654782256686752, + "pass@16": 0.9808689703520912, + "pass@32": 0.9898077910135532, + "pass@64": 0.9945683766692341, + "pass@128": 0.996 + }, + "17": { + "pass@1": 0.7288828125, + "pass@2": 0.8466943897637799, + "pass@4": 0.915012037307836, + "pass@8": 0.9521726149843732, + "pass@16": 0.9726557425898653, + "pass@32": 0.9843864841945104, + "pass@64": 0.9921814457320964, + "pass@128": 0.998 + }, + "18": { + "pass@1": 0.6799375, + "pass@2": 0.8053351377952757, + "pass@4": 0.883184441882265, + "pass@8": 0.9293421197147212, + "pass@16": 0.9568947610573432, + "pass@32": 0.973791820757004, + "pass@64": 0.9839928919990625, + "pass@128": 0.991 + }, + "19": { + "pass@1": 0.6220234375, + "pass@2": 0.7577477854330718, + "pass@4": 0.8491333930446202, + "pass@8": 0.9062156892519055, + "pass@16": 0.942281258892309, + "pass@32": 0.9657194038341252, + "pass@64": 0.9804252054829498, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.5699765625, + "pass@2": 0.70983280019685, + "pass@4": 0.8097918001499816, + "pass@8": 0.8763824013466858, + "pass@16": 0.9191585680481961, + "pass@32": 0.9459104164409721, + "pass@64": 0.9626243327359043, + "pass@128": 0.973 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.8564313322368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8564313322368421, + "pass@2": 0.9151623691980609, + "pass@4": 0.948991970975656, + "pass@8": 0.9687109387412146, + "pass@16": 0.9805335699524597, + "pass@32": 0.987861721133097, + "pass@64": 0.9924805620146625, + "pass@128": 0.9953684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.913015625, + "pass@2": 0.9545599163385837, + "pass@4": 0.9743599124484438, + "pass@8": 0.9845497950289993, + "pass@16": 0.98992591758449, + "pass@32": 0.9929320417591375, + "pass@64": 0.9947494389701538, + "pass@128": 0.996 + }, + "2": { + "pass@1": 0.99959375, + "pass@2": 0.9999697342519684, + "pass@4": 0.9999997261904762, + "pass@8": 0.9999999999828495, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.989890625, + "pass@2": 0.9958089320866145, + "pass@4": 0.9982959692538436, + "pass@8": 0.9993399787031018, + "pass@16": 0.9998187704659531, + "pass@32": 0.9999808388118459, + "pass@64": 0.9999999379485736, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994609375, + "pass@2": 0.9980883366141732, + "pass@4": 0.9990098940757404, + "pass@8": 0.9992710019844361, + "pass@16": 0.9994928857884559, + "pass@32": 0.9997689651291657, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873828125, + "pass@2": 0.992854822834646, + "pass@4": 0.9956864052305961, + "pass@8": 0.9974935802083053, + "pass@16": 0.9986672794278371, + "pass@32": 0.9994369593888157, + "pass@64": 0.9998741889916245, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9779296875, + "pass@2": 0.9891516978346466, + "pass@4": 0.993640552118485, + "pass@8": 0.9963302804712767, + "pass@16": 0.998015205389486, + "pass@32": 0.9992175807153899, + "pass@64": 0.999897068560936, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9710390625, + "pass@2": 0.987645546259844, + "pass@4": 0.9941635037495307, + "pass@8": 0.9969810934042619, + "pass@16": 0.9978741854718784, + "pass@32": 0.9979971178236134, + "pass@64": 0.9979999996026606, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.9629921875, + "pass@2": 0.9842913385826776, + "pass@4": 0.9905712987439065, + "pass@8": 0.9931610401498477, + "pass@16": 0.9947114227372944, + "pass@32": 0.995840688735689, + "pass@64": 0.9966632229237332, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.93903125, + "pass@2": 0.9735146407480327, + "pass@4": 0.9879546507311577, + "pass@8": 0.9946747197488702, + "pass@16": 0.9974400483666008, + "pass@32": 0.9984578344330189, + "pass@64": 0.9988748320897604, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.8736484375, + "pass@2": 0.9301213090551195, + "pass@4": 0.958462992875891, + "pass@8": 0.9746018529563227, + "pass@16": 0.9844361379001465, + "pass@32": 0.9902204108371494, + "pass@64": 0.9933914275171519, + "pass@128": 0.995 + }, + "12": { + "pass@1": 0.865953125, + "pass@2": 0.9204032972440962, + "pass@4": 0.9486816894450689, + "pass@8": 0.9657994139719662, + "pass@16": 0.9770121985407124, + "pass@32": 0.9846340088371232, + "pass@64": 0.9905664884095952, + "pass@128": 0.996 + }, + "13": { + "pass@1": 0.8277890625, + "pass@2": 0.8996813484251981, + "pass@4": 0.938710904293213, + "pass@8": 0.9617088837168377, + "pass@16": 0.9757131222438731, + "pass@32": 0.983740177941991, + "pass@64": 0.9882869729395238, + "pass@128": 0.991 + }, + "14": { + "pass@1": 0.8092734375, + "pass@2": 0.8884940944881912, + "pass@4": 0.9315317574990621, + "pass@8": 0.956514112861527, + "pass@16": 0.973021914723263, + "pass@32": 0.984456875100545, + "pass@64": 0.9919548746245034, + "pass@128": 0.996 + }, + "15": { + "pass@1": 0.7807578125, + "pass@2": 0.8736354576771668, + "pass@4": 0.9264047814960625, + "pass@8": 0.9554910319277518, + "pass@16": 0.9721494395171904, + "pass@32": 0.9830732857749847, + "pass@64": 0.9911087919787471, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.77846875, + "pass@2": 0.8802544291338604, + "pass@4": 0.9362517380014989, + "pass@8": 0.9654782256686752, + "pass@16": 0.9808689703520912, + "pass@32": 0.9898077910135532, + "pass@64": 0.9945683766692341, + "pass@128": 0.996 + }, + "17": { + "pass@1": 0.7288828125, + "pass@2": 0.8466943897637799, + "pass@4": 0.915012037307836, + "pass@8": 0.9521726149843732, + "pass@16": 0.9726557425898653, + "pass@32": 0.9843864841945104, + "pass@64": 0.9921814457320964, + "pass@128": 0.998 + }, + "18": { + "pass@1": 0.6799375, + "pass@2": 0.8053351377952757, + "pass@4": 0.883184441882265, + "pass@8": 0.9293421197147212, + "pass@16": 0.9568947610573432, + "pass@32": 0.973791820757004, + "pass@64": 0.9839928919990625, + "pass@128": 0.991 + }, + "19": { + "pass@1": 0.6220234375, + "pass@2": 0.7577477854330718, + "pass@4": 0.8491333930446202, + "pass@8": 0.9062156892519055, + "pass@16": 0.942281258892309, + "pass@32": 0.9657194038341252, + "pass@64": 0.9804252054829498, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.5699765625, + "pass@2": 0.70983280019685, + "pass@4": 0.8097918001499816, + "pass@8": 0.8763824013466858, + "pass@16": 0.9191585680481961, + "pass@32": 0.9459104164409721, + "pass@64": 0.9626243327359043, + "pass@128": 0.973 + } + }, + "per_op_accuracy": { + "10": 0.913015625, + "2": 0.99959375, + "3": 0.989890625, + "4": 0.994609375, + "5": 0.9873828125, + "6": 0.9779296875, + "7": 0.9710390625, + "8": 0.9629921875, + "9": 0.93903125, + "11": 0.8736484375, + "12": 0.865953125, + "13": 0.8277890625, + "14": 0.8092734375, + "15": 0.7807578125, + "16": 0.77846875, + "17": 0.7288828125, + "18": 0.6799375, + "19": 0.6220234375, + "20": 0.5699765625 + }, + "per_op_avg_response_len": { + "10": 256.4869765625, + "2": 107.1426328125, + "3": 128.4695078125, + "4": 147.98609375, + "5": 170.830171875, + "6": 183.909140625, + "7": 198.936171875, + "8": 223.3921015625, + "9": 241.8748515625, + "11": 277.387671875, + "12": 288.073578125, + "13": 299.479421875, + "14": 328.8324765625, + "15": 344.5523984375, + "16": 371.0299140625, + "17": 382.4528046875, + "18": 396.5237421875, + "19": 407.3039765625, + "20": 424.538359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 256.4869765625, + "2": 107.1426328125, + "3": 128.4695078125, + "4": 147.98609375, + "5": 170.830171875, + "6": 183.909140625, + "7": 198.936171875, + "8": 223.3921015625, + "9": 241.8748515625, + "11": 277.387671875, + "12": 288.073578125, + "13": 299.479421875, + "14": 328.8324765625, + "15": 344.5523984375, + "16": 371.0299140625, + "17": 382.4528046875, + "18": 396.5237421875, + "19": 407.3039765625, + "20": 424.538359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2082841, + "answer_accuracy": 0.8564313322368421, + "avg_response_len": 272.5895785361842, + "resp_tokens_sum": 662937855, + "pass_at_k": { + "pass@1": 0.8564313322368421, + "pass@2": 0.9151623691980609, + "pass@4": 0.948991970975656, + "pass@8": 0.9687109387412146, + "pass@16": 0.9805335699524597, + "pass@32": 0.987861721133097, + "pass@64": 0.9924805620146625, + "pass@128": 0.9953684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.913015625, + "pass@2": 0.9545599163385837, + "pass@4": 0.9743599124484438, + "pass@8": 0.9845497950289993, + "pass@16": 0.98992591758449, + "pass@32": 0.9929320417591375, + "pass@64": 0.9947494389701538, + "pass@128": 0.996 + }, + "2": { + "pass@1": 0.99959375, + "pass@2": 0.9999697342519684, + "pass@4": 0.9999997261904762, + "pass@8": 0.9999999999828495, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.989890625, + "pass@2": 0.9958089320866145, + "pass@4": 0.9982959692538436, + "pass@8": 0.9993399787031018, + "pass@16": 0.9998187704659531, + "pass@32": 0.9999808388118459, + "pass@64": 0.9999999379485736, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994609375, + "pass@2": 0.9980883366141732, + "pass@4": 0.9990098940757404, + "pass@8": 0.9992710019844361, + "pass@16": 0.9994928857884559, + "pass@32": 0.9997689651291657, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873828125, + "pass@2": 0.992854822834646, + "pass@4": 0.9956864052305961, + "pass@8": 0.9974935802083053, + "pass@16": 0.9986672794278371, + "pass@32": 0.9994369593888157, + "pass@64": 0.9998741889916245, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9779296875, + "pass@2": 0.9891516978346466, + "pass@4": 0.993640552118485, + "pass@8": 0.9963302804712767, + "pass@16": 0.998015205389486, + "pass@32": 0.9992175807153899, + "pass@64": 0.999897068560936, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9710390625, + "pass@2": 0.987645546259844, + "pass@4": 0.9941635037495307, + "pass@8": 0.9969810934042619, + "pass@16": 0.9978741854718784, + "pass@32": 0.9979971178236134, + "pass@64": 0.9979999996026606, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.9629921875, + "pass@2": 0.9842913385826776, + "pass@4": 0.9905712987439065, + "pass@8": 0.9931610401498477, + "pass@16": 0.9947114227372944, + "pass@32": 0.995840688735689, + "pass@64": 0.9966632229237332, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.93903125, + "pass@2": 0.9735146407480327, + "pass@4": 0.9879546507311577, + "pass@8": 0.9946747197488702, + "pass@16": 0.9974400483666008, + "pass@32": 0.9984578344330189, + "pass@64": 0.9988748320897604, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.8736484375, + "pass@2": 0.9301213090551195, + "pass@4": 0.958462992875891, + "pass@8": 0.9746018529563227, + "pass@16": 0.9844361379001465, + "pass@32": 0.9902204108371494, + "pass@64": 0.9933914275171519, + "pass@128": 0.995 + }, + "12": { + "pass@1": 0.865953125, + "pass@2": 0.9204032972440962, + "pass@4": 0.9486816894450689, + "pass@8": 0.9657994139719662, + "pass@16": 0.9770121985407124, + "pass@32": 0.9846340088371232, + "pass@64": 0.9905664884095952, + "pass@128": 0.996 + }, + "13": { + "pass@1": 0.8277890625, + "pass@2": 0.8996813484251981, + "pass@4": 0.938710904293213, + "pass@8": 0.9617088837168377, + "pass@16": 0.9757131222438731, + "pass@32": 0.983740177941991, + "pass@64": 0.9882869729395238, + "pass@128": 0.991 + }, + "14": { + "pass@1": 0.8092734375, + "pass@2": 0.8884940944881912, + "pass@4": 0.9315317574990621, + "pass@8": 0.956514112861527, + "pass@16": 0.973021914723263, + "pass@32": 0.984456875100545, + "pass@64": 0.9919548746245034, + "pass@128": 0.996 + }, + "15": { + "pass@1": 0.7807578125, + "pass@2": 0.8736354576771668, + "pass@4": 0.9264047814960625, + "pass@8": 0.9554910319277518, + "pass@16": 0.9721494395171904, + "pass@32": 0.9830732857749847, + "pass@64": 0.9911087919787471, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.77846875, + "pass@2": 0.8802544291338604, + "pass@4": 0.9362517380014989, + "pass@8": 0.9654782256686752, + "pass@16": 0.9808689703520912, + "pass@32": 0.9898077910135532, + "pass@64": 0.9945683766692341, + "pass@128": 0.996 + }, + "17": { + "pass@1": 0.7288828125, + "pass@2": 0.8466943897637799, + "pass@4": 0.915012037307836, + "pass@8": 0.9521726149843732, + "pass@16": 0.9726557425898653, + "pass@32": 0.9843864841945104, + "pass@64": 0.9921814457320964, + "pass@128": 0.998 + }, + "18": { + "pass@1": 0.6799375, + "pass@2": 0.8053351377952757, + "pass@4": 0.883184441882265, + "pass@8": 0.9293421197147212, + "pass@16": 0.9568947610573432, + "pass@32": 0.973791820757004, + "pass@64": 0.9839928919990625, + "pass@128": 0.991 + }, + "19": { + "pass@1": 0.6220234375, + "pass@2": 0.7577477854330718, + "pass@4": 0.8491333930446202, + "pass@8": 0.9062156892519055, + "pass@16": 0.942281258892309, + "pass@32": 0.9657194038341252, + "pass@64": 0.9804252054829498, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.5699765625, + "pass@2": 0.70983280019685, + "pass@4": 0.8097918001499816, + "pass@8": 0.8763824013466858, + "pass@16": 0.9191585680481961, + "pass@32": 0.9459104164409721, + "pass@64": 0.9626243327359043, + "pass@128": 0.973 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5e8597c222b8c71395d75be3ff7070d757cf29d6 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8846a1049ba5ee01b393d7e0ef18993c1f2a7f2e387cc8e94559a02794cbf5e2 +size 2644319180 diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl.graph_results.jsonl b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..61a5becc0ca1ae81d7e6c82250f294cefa758d7e --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29ce03ec2a710c5ad4d79c80d5f6bba10b3c66a1a182e4403c4f26ed258c50c +size 375441059 diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl.graph_results.jsonl b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ffe10b2325e874a60c579e3346f4077568bf9291 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:963a6c53aa9afd4d020fad3130786c556a0ca8a7fbe324c7dd885a6e37fb1e6b +size 429191965 diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5d428c8d0da06454dbf387c87dbdb9d03171bff9 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/graphs", "generated_path": "results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2", "results_dir": "results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs", "per_file": [{"file": "results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 138528, "value_mismatch": 24499, "dependency_mismatch": 897798, "answer_mismatch": 33858, "extra_nodes": 1013021, "missing_nodes": 1013347, "perfect_with_extra": 1, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "dependency_mismatch": 897798, "extra_nodes": 1013021, "missing_nodes": 1013347, "value_mismatch": 24499, "answer_mismatch": 33858, "perfect": 138528, "perfect_with_extra": 1}}, "results_path": "results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 280453, "dependency_mismatch": 1258569, "answer_mismatch": 315301, "extra_nodes": 1279654, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 280453, "dependency_mismatch": 1258569, "answer_mismatch": 315301, "extra_nodes": 1279654, "missing_nodes": 1280000}}, "results_path": "results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 138528, "value_mismatch": 304952, "dependency_mismatch": 2156367, "answer_mismatch": 349159, "extra_nodes": 2292675, "missing_nodes": 2293347, "perfect_with_extra": 1, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "dependency_mismatch": 2156367, "extra_nodes": 2292675, "missing_nodes": 2293347, "value_mismatch": 304952, "answer_mismatch": 349159, "perfect": 138528, "perfect_with_extra": 1}}}} diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/results.json b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..366b4c4d836627b59c14df9981e35e273f6429af --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.98, + "pass@2": 0.993, + "pass@4": 0.997, + "pass@8": 0.998, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.034, + "pass@2": 0.044, + "pass@4": 0.05, + "pass@8": 0.059, + "pass@16": 0.076, + "pass@32": 0.09, + "pass@64": 0.096, + "pass@128": 0.102 + }, + "4": { + "pass@1": 0.024, + "pass@2": 0.026, + "pass@4": 0.033, + "pass@8": 0.038, + "pass@16": 0.039, + "pass@32": 0.044, + "pass@64": 0.047, + "pass@128": 0.048 + }, + "5": { + "pass@1": 0.036, + "pass@2": 0.042, + "pass@4": 0.051, + "pass@8": 0.057, + "pass@16": 0.063, + "pass@32": 0.066, + "pass@64": 0.07, + "pass@128": 0.072 + }, + "6": { + "pass@1": 0.007, + "pass@2": 0.007, + "pass@4": 0.007, + "pass@8": 0.008, + "pass@16": 0.008, + "pass@32": 0.008, + "pass@64": 0.01, + "pass@128": 0.011 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.001, + "pass@128": 0.002 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.98, + "3": 0.034, + "4": 0.024, + "5": 0.036, + "6": 0.007, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.993, + "3": 0.044, + "4": 0.026, + "5": 0.042, + "6": 0.007, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.997, + "3": 0.05, + "4": 0.033, + "5": 0.051, + "6": 0.007, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.998, + "3": 0.059, + "4": 0.038, + "5": 0.057, + "6": 0.008, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.999, + "3": 0.076, + "4": 0.039, + "5": 0.063, + "6": 0.008, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.999, + "3": 0.09, + "4": 0.044, + "5": 0.066, + "6": 0.008, + "7": 0.0, + "8": 0.0, + "9": 0.001, + "10": 0.0 + }, + "pass@64": { + "2": 0.999, + "3": 0.096, + "4": 0.047, + "5": 0.07, + "6": 0.01, + "7": 0.0, + "8": 0.0, + "9": 0.001, + "10": 0.0 + }, + "pass@128": { + "2": 0.999, + "3": 0.102, + "4": 0.048, + "5": 0.072, + "6": 0.011, + "7": 0.0, + "8": 0.0, + "9": 0.002, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.12011111111111111, + "pass@2": 0.12355555555555556, + "pass@4": 0.12644444444444444, + "pass@8": 0.1288888888888889, + "pass@16": 0.13166666666666665, + "pass@32": 0.13422222222222221, + "pass@64": 0.1358888888888889, + "pass@128": 0.1371111111111111 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.98, + "pass@2": 0.993, + "pass@4": 0.997, + "pass@8": 0.998, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.034, + "pass@2": 0.044, + "pass@4": 0.05, + "pass@8": 0.059, + "pass@16": 0.076, + "pass@32": 0.09, + "pass@64": 0.096, + "pass@128": 0.102 + }, + "4": { + "pass@1": 0.024, + "pass@2": 0.026, + "pass@4": 0.033, + "pass@8": 0.038, + "pass@16": 0.039, + "pass@32": 0.044, + "pass@64": 0.047, + "pass@128": 0.048 + }, + "5": { + "pass@1": 0.036, + "pass@2": 0.042, + "pass@4": 0.051, + "pass@8": 0.057, + "pass@16": 0.063, + "pass@32": 0.066, + "pass@64": 0.07, + "pass@128": 0.072 + }, + "6": { + "pass@1": 0.007, + "pass@2": 0.007, + "pass@4": 0.007, + "pass@8": 0.008, + "pass@16": 0.008, + "pass@32": 0.008, + "pass@64": 0.01, + "pass@128": 0.011 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.001, + "pass@128": 0.002 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.98, + "3": 0.034, + "4": 0.024, + "5": 0.036, + "6": 0.007, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.993, + "3": 0.044, + "4": 0.026, + "5": 0.042, + "6": 0.007, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.997, + "3": 0.05, + "4": 0.033, + "5": 0.051, + "6": 0.007, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.998, + "3": 0.059, + "4": 0.038, + "5": 0.057, + "6": 0.008, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.999, + "3": 0.076, + "4": 0.039, + "5": 0.063, + "6": 0.008, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.999, + "3": 0.09, + "4": 0.044, + "5": 0.066, + "6": 0.008, + "7": 0.0, + "8": 0.0, + "9": 0.001, + "10": 0.0 + }, + "pass@64": { + "2": 0.999, + "3": 0.096, + "4": 0.047, + "5": 0.07, + "6": 0.01, + "7": 0.0, + "8": 0.0, + "9": 0.001, + "10": 0.0 + }, + "pass@128": { + "2": 0.999, + "3": 0.102, + "4": 0.048, + "5": 0.072, + "6": 0.011, + "7": 0.0, + "8": 0.0, + "9": 0.002, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.12011111111111111, + "pass@2": 0.12355555555555556, + "pass@4": 0.12644444444444444, + "pass@8": 0.1288888888888889, + "pass@16": 0.13166666666666665, + "pass@32": 0.13422222222222221, + "pass@64": 0.1358888888888889, + "pass@128": 0.1371111111111111 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/summary.csv b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a37449e79a0198c3197d51ae72772ed33b7f2fe8 --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2,0.970609375,0.75367109375,0.8564313322368421,0.0,0.0,0.0,184.33640538194445,352.017434375,1152000,1280000,2432000 diff --git a/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/summary.json b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2974cdb2b679356efbd9ac51422a5d812406616e --- /dev/null +++ b/results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2", + "id_acc": 0.970609375, + "ood_acc": 0.75367109375, + "total_acc": 0.8564313322368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 184.33640538194445, + "ood_avg_resp_len": 352.017434375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2082841, + "answer_accuracy": 0.8564313322368421, + "avg_response_len": 272.5895785361842, + "resp_tokens_sum": 662937855, + "pass_at_k": { + "pass@1": 0.8564313322368421, + "pass@2": 0.9151623691980609, + "pass@4": 0.948991970975656, + "pass@8": 0.9687109387412146, + "pass@16": 0.9805335699524597, + "pass@32": 0.987861721133097, + "pass@64": 0.9924805620146625, + "pass@128": 0.9953684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.913015625, + "pass@2": 0.9545599163385837, + "pass@4": 0.9743599124484438, + "pass@8": 0.9845497950289993, + "pass@16": 0.98992591758449, + "pass@32": 0.9929320417591375, + "pass@64": 0.9947494389701538, + "pass@128": 0.996 + }, + "2": { + "pass@1": 0.99959375, + "pass@2": 0.9999697342519684, + "pass@4": 0.9999997261904762, + "pass@8": 0.9999999999828495, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.989890625, + "pass@2": 0.9958089320866145, + "pass@4": 0.9982959692538436, + "pass@8": 0.9993399787031018, + "pass@16": 0.9998187704659531, + "pass@32": 0.9999808388118459, + "pass@64": 0.9999999379485736, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994609375, + "pass@2": 0.9980883366141732, + "pass@4": 0.9990098940757404, + "pass@8": 0.9992710019844361, + "pass@16": 0.9994928857884559, + "pass@32": 0.9997689651291657, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873828125, + "pass@2": 0.992854822834646, + "pass@4": 0.9956864052305961, + "pass@8": 0.9974935802083053, + "pass@16": 0.9986672794278371, + "pass@32": 0.9994369593888157, + "pass@64": 0.9998741889916245, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9779296875, + "pass@2": 0.9891516978346466, + "pass@4": 0.993640552118485, + "pass@8": 0.9963302804712767, + "pass@16": 0.998015205389486, + "pass@32": 0.9992175807153899, + "pass@64": 0.999897068560936, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9710390625, + "pass@2": 0.987645546259844, + "pass@4": 0.9941635037495307, + "pass@8": 0.9969810934042619, + "pass@16": 0.9978741854718784, + "pass@32": 0.9979971178236134, + "pass@64": 0.9979999996026606, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.9629921875, + "pass@2": 0.9842913385826776, + "pass@4": 0.9905712987439065, + "pass@8": 0.9931610401498477, + "pass@16": 0.9947114227372944, + "pass@32": 0.995840688735689, + "pass@64": 0.9966632229237332, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.93903125, + "pass@2": 0.9735146407480327, + "pass@4": 0.9879546507311577, + "pass@8": 0.9946747197488702, + "pass@16": 0.9974400483666008, + "pass@32": 0.9984578344330189, + "pass@64": 0.9988748320897604, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.8736484375, + "pass@2": 0.9301213090551195, + "pass@4": 0.958462992875891, + "pass@8": 0.9746018529563227, + "pass@16": 0.9844361379001465, + "pass@32": 0.9902204108371494, + "pass@64": 0.9933914275171519, + "pass@128": 0.995 + }, + "12": { + "pass@1": 0.865953125, + "pass@2": 0.9204032972440962, + "pass@4": 0.9486816894450689, + "pass@8": 0.9657994139719662, + "pass@16": 0.9770121985407124, + "pass@32": 0.9846340088371232, + "pass@64": 0.9905664884095952, + "pass@128": 0.996 + }, + "13": { + "pass@1": 0.8277890625, + "pass@2": 0.8996813484251981, + "pass@4": 0.938710904293213, + "pass@8": 0.9617088837168377, + "pass@16": 0.9757131222438731, + "pass@32": 0.983740177941991, + "pass@64": 0.9882869729395238, + "pass@128": 0.991 + }, + "14": { + "pass@1": 0.8092734375, + "pass@2": 0.8884940944881912, + "pass@4": 0.9315317574990621, + "pass@8": 0.956514112861527, + "pass@16": 0.973021914723263, + "pass@32": 0.984456875100545, + "pass@64": 0.9919548746245034, + "pass@128": 0.996 + }, + "15": { + "pass@1": 0.7807578125, + "pass@2": 0.8736354576771668, + "pass@4": 0.9264047814960625, + "pass@8": 0.9554910319277518, + "pass@16": 0.9721494395171904, + "pass@32": 0.9830732857749847, + "pass@64": 0.9911087919787471, + "pass@128": 0.996 + }, + "16": { + "pass@1": 0.77846875, + "pass@2": 0.8802544291338604, + "pass@4": 0.9362517380014989, + "pass@8": 0.9654782256686752, + "pass@16": 0.9808689703520912, + "pass@32": 0.9898077910135532, + "pass@64": 0.9945683766692341, + "pass@128": 0.996 + }, + "17": { + "pass@1": 0.7288828125, + "pass@2": 0.8466943897637799, + "pass@4": 0.915012037307836, + "pass@8": 0.9521726149843732, + "pass@16": 0.9726557425898653, + "pass@32": 0.9843864841945104, + "pass@64": 0.9921814457320964, + "pass@128": 0.998 + }, + "18": { + "pass@1": 0.6799375, + "pass@2": 0.8053351377952757, + "pass@4": 0.883184441882265, + "pass@8": 0.9293421197147212, + "pass@16": 0.9568947610573432, + "pass@32": 0.973791820757004, + "pass@64": 0.9839928919990625, + "pass@128": 0.991 + }, + "19": { + "pass@1": 0.6220234375, + "pass@2": 0.7577477854330718, + "pass@4": 0.8491333930446202, + "pass@8": 0.9062156892519055, + "pass@16": 0.942281258892309, + "pass@32": 0.9657194038341252, + "pass@64": 0.9804252054829498, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.5699765625, + "pass@2": 0.70983280019685, + "pass@4": 0.8097918001499816, + "pass@8": 0.8763824013466858, + "pass@16": 0.9191585680481961, + "pass@32": 0.9459104164409721, + "pass@64": 0.9626243327359043, + "pass@128": 0.973 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..06e365cfe947a6077b3a5a80659a345449d62523 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b2266a4a2ea70156b24c89c1338131595898d0bf8f0f10ef76aa170d5206060 +size 1574633897 diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_metrics.json b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..faba9b0280d71965432cdbdeaa826e93f8e15ced --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm", + "id": { + "answer_accuracy": 0.39958767361111114, + "avg_response_len": 140.6659592013889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.39958767361111114, + "pass@2": 0.4373426017060438, + "pass@4": 0.4739806857372012, + "pass@8": 0.5117075710444569, + "pass@16": 0.5507645606115887, + "pass@32": 0.5898780604974418, + "pass@64": 0.6273680584970976, + "pass@128": 0.6616666666666666 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1830546875, + "pass@2": 0.20952116141732297, + "pass@4": 0.2383966255155605, + "pass@8": 0.2710427312210943, + "pass@16": 0.30699981235999735, + "pass@32": 0.34551654257693415, + "pass@64": 0.38587587679343294, + "pass@128": 0.424 + }, + "2": { + "pass@1": 0.9814609375, + "pass@2": 0.9922114911417323, + "pass@4": 0.9968508753280841, + "pass@8": 0.9988351225906016, + "pass@16": 0.9996941647558666, + "pass@32": 0.9999732028481125, + "pass@64": 0.9999999327042958, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.845484375, + "pass@2": 0.8951834399606313, + "pass@4": 0.9228552554368202, + "pass@8": 0.9404669319809784, + "pass@16": 0.9529833520371419, + "pass@32": 0.961181431816883, + "pass@64": 0.9660464891138485, + "pass@128": 0.969 + }, + "4": { + "pass@1": 0.333078125, + "pass@2": 0.3811968503937009, + "pass@4": 0.4293913296775398, + "pass@8": 0.47646773798502506, + "pass@16": 0.5206730324364229, + "pass@32": 0.5604681683428177, + "pass@64": 0.5957565277950225, + "pass@128": 0.629 + }, + "5": { + "pass@1": 0.29678125, + "pass@2": 0.34330930118110237, + "pass@4": 0.3887239339145104, + "pass@8": 0.43441612237924027, + "pass@16": 0.4809806955064174, + "pass@32": 0.5271614616427763, + "pass@64": 0.5701744077349079, + "pass@128": 0.607 + }, + "6": { + "pass@1": 0.3149921875, + "pass@2": 0.36090834153543305, + "pass@4": 0.40982087579677534, + "pass@8": 0.46266170600441797, + "pass@16": 0.5185506359130283, + "pass@32": 0.5734374683781633, + "pass@64": 0.6231664530144211, + "pass@128": 0.665 + }, + "7": { + "pass@1": 0.2305, + "pass@2": 0.27067015255905474, + "pass@4": 0.3152229831271087, + "pass@8": 0.3643300065773014, + "pass@16": 0.41698232473123015, + "pass@32": 0.4714699999776515, + "pass@64": 0.5240621552910725, + "pass@128": 0.572 + }, + "8": { + "pass@1": 0.223328125, + "pass@2": 0.26172244094488195, + "pass@4": 0.3050642823397073, + "pass@8": 0.3531824798442372, + "pass@16": 0.40548544167153877, + "pass@32": 0.46207481159213004, + "pass@64": 0.5218905962944446, + "pass@128": 0.581 + }, + "9": { + "pass@1": 0.187609375, + "pass@2": 0.22136023622047235, + "pass@4": 0.25950001049868743, + "pass@8": 0.3039653008171441, + "pass@16": 0.35453158609254953, + "pass@32": 0.40761945730152294, + "pass@64": 0.459340087732421, + "pass@128": 0.508 + } + }, + "per_op_accuracy": { + "10": 0.1830546875, + "2": 0.9814609375, + "3": 0.845484375, + "4": 0.333078125, + "5": 0.29678125, + "6": 0.3149921875, + "7": 0.2305, + "8": 0.223328125, + "9": 0.187609375 + }, + "per_op_avg_response_len": { + "10": 154.7713671875, + "2": 108.9817734375, + "3": 114.4234921875, + "4": 128.3030546875, + "5": 143.71990625, + "6": 150.02684375, + "7": 146.78134375, + "8": 157.5693984375, + "9": 161.416453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 154.7713671875, + "2": 108.9817734375, + "3": 114.4234921875, + "4": 128.3030546875, + "5": 143.71990625, + "6": 150.02684375, + "7": 146.78134375, + "8": 157.5693984375, + "9": 161.416453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 460325, + "answer_accuracy": 0.39958767361111114, + "avg_response_len": 140.6659592013889, + "resp_tokens_sum": 162047185, + "pass_at_k": { + "pass@1": 0.39958767361111114, + "pass@2": 0.4373426017060438, + "pass@4": 0.4739806857372012, + "pass@8": 0.5117075710444569, + "pass@16": 0.5507645606115887, + "pass@32": 0.5898780604974418, + "pass@64": 0.6273680584970976, + "pass@128": 0.6616666666666666 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1830546875, + "pass@2": 0.20952116141732297, + "pass@4": 0.2383966255155605, + "pass@8": 0.2710427312210943, + "pass@16": 0.30699981235999735, + "pass@32": 0.34551654257693415, + "pass@64": 0.38587587679343294, + "pass@128": 0.424 + }, + "2": { + "pass@1": 0.9814609375, + "pass@2": 0.9922114911417323, + "pass@4": 0.9968508753280841, + "pass@8": 0.9988351225906016, + "pass@16": 0.9996941647558666, + "pass@32": 0.9999732028481125, + "pass@64": 0.9999999327042958, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.845484375, + "pass@2": 0.8951834399606313, + "pass@4": 0.9228552554368202, + "pass@8": 0.9404669319809784, + "pass@16": 0.9529833520371419, + "pass@32": 0.961181431816883, + "pass@64": 0.9660464891138485, + "pass@128": 0.969 + }, + "4": { + "pass@1": 0.333078125, + "pass@2": 0.3811968503937009, + "pass@4": 0.4293913296775398, + "pass@8": 0.47646773798502506, + "pass@16": 0.5206730324364229, + "pass@32": 0.5604681683428177, + "pass@64": 0.5957565277950225, + "pass@128": 0.629 + }, + "5": { + "pass@1": 0.29678125, + "pass@2": 0.34330930118110237, + "pass@4": 0.3887239339145104, + "pass@8": 0.43441612237924027, + "pass@16": 0.4809806955064174, + "pass@32": 0.5271614616427763, + "pass@64": 0.5701744077349079, + "pass@128": 0.607 + }, + "6": { + "pass@1": 0.3149921875, + "pass@2": 0.36090834153543305, + "pass@4": 0.40982087579677534, + "pass@8": 0.46266170600441797, + "pass@16": 0.5185506359130283, + "pass@32": 0.5734374683781633, + "pass@64": 0.6231664530144211, + "pass@128": 0.665 + }, + "7": { + "pass@1": 0.2305, + "pass@2": 0.27067015255905474, + "pass@4": 0.3152229831271087, + "pass@8": 0.3643300065773014, + "pass@16": 0.41698232473123015, + "pass@32": 0.4714699999776515, + "pass@64": 0.5240621552910725, + "pass@128": 0.572 + }, + "8": { + "pass@1": 0.223328125, + "pass@2": 0.26172244094488195, + "pass@4": 0.3050642823397073, + "pass@8": 0.3531824798442372, + "pass@16": 0.40548544167153877, + "pass@32": 0.46207481159213004, + "pass@64": 0.5218905962944446, + "pass@128": 0.581 + }, + "9": { + "pass@1": 0.187609375, + "pass@2": 0.22136023622047235, + "pass@4": 0.25950001049868743, + "pass@8": 0.3039653008171441, + "pass@16": 0.35453158609254953, + "pass@32": 0.40761945730152294, + "pass@64": 0.459340087732421, + "pass@128": 0.508 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.14737265625, + "avg_response_len": 164.050321875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.14737265625, + "pass@2": 0.17112230561023734, + "pass@4": 0.1946401743250852, + "pass@8": 0.21920288690802966, + "pass@16": 0.24564145321096925, + "pass@32": 0.27465608474079606, + "pass@64": 0.30688712751998304, + "pass@128": 0.3428 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.183015625, + "pass@2": 0.20866338582677174, + "pass@4": 0.23590405080614907, + "pass@8": 0.26657177499164364, + "pass@16": 0.3005953479595448, + "pass@32": 0.3384648822247731, + "pass@64": 0.38032339726143644, + "pass@128": 0.426 + }, + "12": { + "pass@1": 0.178078125, + "pass@2": 0.201375246062992, + "pass@4": 0.22511521456692912, + "pass@8": 0.2512357236143783, + "pass@16": 0.28027664483296916, + "pass@32": 0.31188903107657634, + "pass@64": 0.3440739098252981, + "pass@128": 0.377 + }, + "13": { + "pass@1": 0.13940625, + "pass@2": 0.16364333169291342, + "pass@4": 0.18814229461942247, + "pass@8": 0.21404766874082337, + "pass@16": 0.2409017702074673, + "pass@32": 0.26905766865909836, + "pass@64": 0.3004190790273317, + "pass@128": 0.336 + }, + "14": { + "pass@1": 0.15340625, + "pass@2": 0.1768810285433071, + "pass@4": 0.2000666328271464, + "pass@8": 0.22366324385024125, + "pass@16": 0.24802844539120988, + "pass@32": 0.27423768794616893, + "pass@64": 0.3050598647050578, + "pass@128": 0.342 + }, + "15": { + "pass@1": 0.1334140625, + "pass@2": 0.1554541092519686, + "pass@4": 0.17874117154105718, + "pass@8": 0.20425971936038342, + "pass@16": 0.2329328406948268, + "pass@32": 0.2661451656254728, + "pass@64": 0.30418256883970834, + "pass@128": 0.346 + }, + "16": { + "pass@1": 0.1628203125, + "pass@2": 0.18587364665354322, + "pass@4": 0.207851382358455, + "pass@8": 0.2305969739830245, + "pass@16": 0.25480601080377036, + "pass@32": 0.2811469496260871, + "pass@64": 0.31011607240542965, + "pass@128": 0.342 + }, + "17": { + "pass@1": 0.143125, + "pass@2": 0.16872834645669282, + "pass@4": 0.19324112617172834, + "pass@8": 0.21683406747756903, + "pass@16": 0.24070540026092216, + "pass@32": 0.26551813331097773, + "pass@64": 0.2924959059932723, + "pass@128": 0.324 + }, + "18": { + "pass@1": 0.126140625, + "pass@2": 0.1503442421259842, + "pass@4": 0.17417070163104598, + "pass@8": 0.19968930018568581, + "pass@16": 0.2286068333990339, + "pass@32": 0.2611928633681317, + "pass@64": 0.2964590683968482, + "pass@128": 0.332 + }, + "19": { + "pass@1": 0.135515625, + "pass@2": 0.15826587106299214, + "pass@4": 0.17993976509186338, + "pass@8": 0.20132565635563035, + "pass@16": 0.2238203540907909, + "pass@32": 0.24772720608105048, + "pass@64": 0.27351685907521717, + "pass@128": 0.302 + }, + "20": { + "pass@1": 0.1188046875, + "pass@2": 0.14199384842519686, + "pass@4": 0.16322940363704533, + "pass@8": 0.18380474052090803, + "pass@16": 0.20574088446916178, + "pass@32": 0.23118125948963936, + "pass@64": 0.26222454967024666, + "pass@128": 0.301 + } + }, + "per_op_accuracy": { + "11": 0.183015625, + "12": 0.178078125, + "13": 0.13940625, + "14": 0.15340625, + "15": 0.1334140625, + "16": 0.1628203125, + "17": 0.143125, + "18": 0.126140625, + "19": 0.135515625, + "20": 0.1188046875 + }, + "per_op_avg_response_len": { + "11": 161.096625, + "12": 155.0810625, + "13": 153.9815, + "14": 161.669921875, + "15": 164.9374296875, + "16": 167.152625, + "17": 170.78396875, + "18": 172.1340078125, + "19": 164.541390625, + "20": 169.1246875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 161.096625, + "12": 155.0810625, + "13": 153.9815, + "14": 161.669921875, + "15": 164.9374296875, + "16": 167.152625, + "17": 170.78396875, + "18": 172.1340078125, + "19": 164.541390625, + "20": 169.1246875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 188637, + "answer_accuracy": 0.14737265625, + "avg_response_len": 164.050321875, + "resp_tokens_sum": 209984412, + "pass_at_k": { + "pass@1": 0.14737265625, + "pass@2": 0.17112230561023734, + "pass@4": 0.1946401743250852, + "pass@8": 0.21920288690802966, + "pass@16": 0.24564145321096925, + "pass@32": 0.27465608474079606, + "pass@64": 0.30688712751998304, + "pass@128": 0.3428 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.183015625, + "pass@2": 0.20866338582677174, + "pass@4": 0.23590405080614907, + "pass@8": 0.26657177499164364, + "pass@16": 0.3005953479595448, + "pass@32": 0.3384648822247731, + "pass@64": 0.38032339726143644, + "pass@128": 0.426 + }, + "12": { + "pass@1": 0.178078125, + "pass@2": 0.201375246062992, + "pass@4": 0.22511521456692912, + "pass@8": 0.2512357236143783, + "pass@16": 0.28027664483296916, + "pass@32": 0.31188903107657634, + "pass@64": 0.3440739098252981, + "pass@128": 0.377 + }, + "13": { + "pass@1": 0.13940625, + "pass@2": 0.16364333169291342, + "pass@4": 0.18814229461942247, + "pass@8": 0.21404766874082337, + "pass@16": 0.2409017702074673, + "pass@32": 0.26905766865909836, + "pass@64": 0.3004190790273317, + "pass@128": 0.336 + }, + "14": { + "pass@1": 0.15340625, + "pass@2": 0.1768810285433071, + "pass@4": 0.2000666328271464, + "pass@8": 0.22366324385024125, + "pass@16": 0.24802844539120988, + "pass@32": 0.27423768794616893, + "pass@64": 0.3050598647050578, + "pass@128": 0.342 + }, + "15": { + "pass@1": 0.1334140625, + "pass@2": 0.1554541092519686, + "pass@4": 0.17874117154105718, + "pass@8": 0.20425971936038342, + "pass@16": 0.2329328406948268, + "pass@32": 0.2661451656254728, + "pass@64": 0.30418256883970834, + "pass@128": 0.346 + }, + "16": { + "pass@1": 0.1628203125, + "pass@2": 0.18587364665354322, + "pass@4": 0.207851382358455, + "pass@8": 0.2305969739830245, + "pass@16": 0.25480601080377036, + "pass@32": 0.2811469496260871, + "pass@64": 0.31011607240542965, + "pass@128": 0.342 + }, + "17": { + "pass@1": 0.143125, + "pass@2": 0.16872834645669282, + "pass@4": 0.19324112617172834, + "pass@8": 0.21683406747756903, + "pass@16": 0.24070540026092216, + "pass@32": 0.26551813331097773, + "pass@64": 0.2924959059932723, + "pass@128": 0.324 + }, + "18": { + "pass@1": 0.126140625, + "pass@2": 0.1503442421259842, + "pass@4": 0.17417070163104598, + "pass@8": 0.19968930018568581, + "pass@16": 0.2286068333990339, + "pass@32": 0.2611928633681317, + "pass@64": 0.2964590683968482, + "pass@128": 0.332 + }, + "19": { + "pass@1": 0.135515625, + "pass@2": 0.15826587106299214, + "pass@4": 0.17993976509186338, + "pass@8": 0.20132565635563035, + "pass@16": 0.2238203540907909, + "pass@32": 0.24772720608105048, + "pass@64": 0.27351685907521717, + "pass@128": 0.302 + }, + "20": { + "pass@1": 0.1188046875, + "pass@2": 0.14199384842519686, + "pass@4": 0.16322940363704533, + "pass@8": 0.18380474052090803, + "pass@16": 0.20574088446916178, + "pass@32": 0.23118125948963936, + "pass@64": 0.26222454967024666, + "pass@128": 0.301 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.26684292763157896, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.26684292763157896, + "pass@2": 0.29722665639245166, + "pass@4": 0.32695936394134406, + "pass@8": 0.357757737288442, + "pass@16": 0.39017345145337295, + "pass@32": 0.42397175746762866, + "pass@64": 0.45869388429861935, + "pass@128": 0.4938421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1830546875, + "pass@2": 0.20952116141732297, + "pass@4": 0.2383966255155605, + "pass@8": 0.2710427312210943, + "pass@16": 0.30699981235999735, + "pass@32": 0.34551654257693415, + "pass@64": 0.38587587679343294, + "pass@128": 0.424 + }, + "2": { + "pass@1": 0.9814609375, + "pass@2": 0.9922114911417323, + "pass@4": 0.9968508753280841, + "pass@8": 0.9988351225906016, + "pass@16": 0.9996941647558666, + "pass@32": 0.9999732028481125, + "pass@64": 0.9999999327042958, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.845484375, + "pass@2": 0.8951834399606313, + "pass@4": 0.9228552554368202, + "pass@8": 0.9404669319809784, + "pass@16": 0.9529833520371419, + "pass@32": 0.961181431816883, + "pass@64": 0.9660464891138485, + "pass@128": 0.969 + }, + "4": { + "pass@1": 0.333078125, + "pass@2": 0.3811968503937009, + "pass@4": 0.4293913296775398, + "pass@8": 0.47646773798502506, + "pass@16": 0.5206730324364229, + "pass@32": 0.5604681683428177, + "pass@64": 0.5957565277950225, + "pass@128": 0.629 + }, + "5": { + "pass@1": 0.29678125, + "pass@2": 0.34330930118110237, + "pass@4": 0.3887239339145104, + "pass@8": 0.43441612237924027, + "pass@16": 0.4809806955064174, + "pass@32": 0.5271614616427763, + "pass@64": 0.5701744077349079, + "pass@128": 0.607 + }, + "6": { + "pass@1": 0.3149921875, + "pass@2": 0.36090834153543305, + "pass@4": 0.40982087579677534, + "pass@8": 0.46266170600441797, + "pass@16": 0.5185506359130283, + "pass@32": 0.5734374683781633, + "pass@64": 0.6231664530144211, + "pass@128": 0.665 + }, + "7": { + "pass@1": 0.2305, + "pass@2": 0.27067015255905474, + "pass@4": 0.3152229831271087, + "pass@8": 0.3643300065773014, + "pass@16": 0.41698232473123015, + "pass@32": 0.4714699999776515, + "pass@64": 0.5240621552910725, + "pass@128": 0.572 + }, + "8": { + "pass@1": 0.223328125, + "pass@2": 0.26172244094488195, + "pass@4": 0.3050642823397073, + "pass@8": 0.3531824798442372, + "pass@16": 0.40548544167153877, + "pass@32": 0.46207481159213004, + "pass@64": 0.5218905962944446, + "pass@128": 0.581 + }, + "9": { + "pass@1": 0.187609375, + "pass@2": 0.22136023622047235, + "pass@4": 0.25950001049868743, + "pass@8": 0.3039653008171441, + "pass@16": 0.35453158609254953, + "pass@32": 0.40761945730152294, + "pass@64": 0.459340087732421, + "pass@128": 0.508 + }, + "11": { + "pass@1": 0.183015625, + "pass@2": 0.20866338582677174, + "pass@4": 0.23590405080614907, + "pass@8": 0.26657177499164364, + "pass@16": 0.3005953479595448, + "pass@32": 0.3384648822247731, + "pass@64": 0.38032339726143644, + "pass@128": 0.426 + }, + "12": { + "pass@1": 0.178078125, + "pass@2": 0.201375246062992, + "pass@4": 0.22511521456692912, + "pass@8": 0.2512357236143783, + "pass@16": 0.28027664483296916, + "pass@32": 0.31188903107657634, + "pass@64": 0.3440739098252981, + "pass@128": 0.377 + }, + "13": { + "pass@1": 0.13940625, + "pass@2": 0.16364333169291342, + "pass@4": 0.18814229461942247, + "pass@8": 0.21404766874082337, + "pass@16": 0.2409017702074673, + "pass@32": 0.26905766865909836, + "pass@64": 0.3004190790273317, + "pass@128": 0.336 + }, + "14": { + "pass@1": 0.15340625, + "pass@2": 0.1768810285433071, + "pass@4": 0.2000666328271464, + "pass@8": 0.22366324385024125, + "pass@16": 0.24802844539120988, + "pass@32": 0.27423768794616893, + "pass@64": 0.3050598647050578, + "pass@128": 0.342 + }, + "15": { + "pass@1": 0.1334140625, + "pass@2": 0.1554541092519686, + "pass@4": 0.17874117154105718, + "pass@8": 0.20425971936038342, + "pass@16": 0.2329328406948268, + "pass@32": 0.2661451656254728, + "pass@64": 0.30418256883970834, + "pass@128": 0.346 + }, + "16": { + "pass@1": 0.1628203125, + "pass@2": 0.18587364665354322, + "pass@4": 0.207851382358455, + "pass@8": 0.2305969739830245, + "pass@16": 0.25480601080377036, + "pass@32": 0.2811469496260871, + "pass@64": 0.31011607240542965, + "pass@128": 0.342 + }, + "17": { + "pass@1": 0.143125, + "pass@2": 0.16872834645669282, + "pass@4": 0.19324112617172834, + "pass@8": 0.21683406747756903, + "pass@16": 0.24070540026092216, + "pass@32": 0.26551813331097773, + "pass@64": 0.2924959059932723, + "pass@128": 0.324 + }, + "18": { + "pass@1": 0.126140625, + "pass@2": 0.1503442421259842, + "pass@4": 0.17417070163104598, + "pass@8": 0.19968930018568581, + "pass@16": 0.2286068333990339, + "pass@32": 0.2611928633681317, + "pass@64": 0.2964590683968482, + "pass@128": 0.332 + }, + "19": { + "pass@1": 0.135515625, + "pass@2": 0.15826587106299214, + "pass@4": 0.17993976509186338, + "pass@8": 0.20132565635563035, + "pass@16": 0.2238203540907909, + "pass@32": 0.24772720608105048, + "pass@64": 0.27351685907521717, + "pass@128": 0.302 + }, + "20": { + "pass@1": 0.1188046875, + "pass@2": 0.14199384842519686, + "pass@4": 0.16322940363704533, + "pass@8": 0.18380474052090803, + "pass@16": 0.20574088446916178, + "pass@32": 0.23118125948963936, + "pass@64": 0.26222454967024666, + "pass@128": 0.301 + } + }, + "per_op_accuracy": { + "10": 0.1830546875, + "2": 0.9814609375, + "3": 0.845484375, + "4": 0.333078125, + "5": 0.29678125, + "6": 0.3149921875, + "7": 0.2305, + "8": 0.223328125, + "9": 0.187609375, + "11": 0.183015625, + "12": 0.178078125, + "13": 0.13940625, + "14": 0.15340625, + "15": 0.1334140625, + "16": 0.1628203125, + "17": 0.143125, + "18": 0.126140625, + "19": 0.135515625, + "20": 0.1188046875 + }, + "per_op_avg_response_len": { + "10": 154.7713671875, + "2": 108.9817734375, + "3": 114.4234921875, + "4": 128.3030546875, + "5": 143.71990625, + "6": 150.02684375, + "7": 146.78134375, + "8": 157.5693984375, + "9": 161.416453125, + "11": 161.096625, + "12": 155.0810625, + "13": 153.9815, + "14": 161.669921875, + "15": 164.9374296875, + "16": 167.152625, + "17": 170.78396875, + "18": 172.1340078125, + "19": 164.541390625, + "20": 169.1246875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 154.7713671875, + "2": 108.9817734375, + "3": 114.4234921875, + "4": 128.3030546875, + "5": 143.71990625, + "6": 150.02684375, + "7": 146.78134375, + "8": 157.5693984375, + "9": 161.416453125, + "11": 161.096625, + "12": 155.0810625, + "13": 153.9815, + "14": 161.669921875, + "15": 164.9374296875, + "16": 167.152625, + "17": 170.78396875, + "18": 172.1340078125, + "19": 164.541390625, + "20": 169.1246875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 648962, + "answer_accuracy": 0.26684292763157896, + "avg_response_len": 152.97351850328948, + "resp_tokens_sum": 372031597, + "pass_at_k": { + "pass@1": 0.26684292763157896, + "pass@2": 0.29722665639245166, + "pass@4": 0.32695936394134406, + "pass@8": 0.357757737288442, + "pass@16": 0.39017345145337295, + "pass@32": 0.42397175746762866, + "pass@64": 0.45869388429861935, + "pass@128": 0.4938421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1830546875, + "pass@2": 0.20952116141732297, + "pass@4": 0.2383966255155605, + "pass@8": 0.2710427312210943, + "pass@16": 0.30699981235999735, + "pass@32": 0.34551654257693415, + "pass@64": 0.38587587679343294, + "pass@128": 0.424 + }, + "2": { + "pass@1": 0.9814609375, + "pass@2": 0.9922114911417323, + "pass@4": 0.9968508753280841, + "pass@8": 0.9988351225906016, + "pass@16": 0.9996941647558666, + "pass@32": 0.9999732028481125, + "pass@64": 0.9999999327042958, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.845484375, + "pass@2": 0.8951834399606313, + "pass@4": 0.9228552554368202, + "pass@8": 0.9404669319809784, + "pass@16": 0.9529833520371419, + "pass@32": 0.961181431816883, + "pass@64": 0.9660464891138485, + "pass@128": 0.969 + }, + "4": { + "pass@1": 0.333078125, + "pass@2": 0.3811968503937009, + "pass@4": 0.4293913296775398, + "pass@8": 0.47646773798502506, + "pass@16": 0.5206730324364229, + "pass@32": 0.5604681683428177, + "pass@64": 0.5957565277950225, + "pass@128": 0.629 + }, + "5": { + "pass@1": 0.29678125, + "pass@2": 0.34330930118110237, + "pass@4": 0.3887239339145104, + "pass@8": 0.43441612237924027, + "pass@16": 0.4809806955064174, + "pass@32": 0.5271614616427763, + "pass@64": 0.5701744077349079, + "pass@128": 0.607 + }, + "6": { + "pass@1": 0.3149921875, + "pass@2": 0.36090834153543305, + "pass@4": 0.40982087579677534, + "pass@8": 0.46266170600441797, + "pass@16": 0.5185506359130283, + "pass@32": 0.5734374683781633, + "pass@64": 0.6231664530144211, + "pass@128": 0.665 + }, + "7": { + "pass@1": 0.2305, + "pass@2": 0.27067015255905474, + "pass@4": 0.3152229831271087, + "pass@8": 0.3643300065773014, + "pass@16": 0.41698232473123015, + "pass@32": 0.4714699999776515, + "pass@64": 0.5240621552910725, + "pass@128": 0.572 + }, + "8": { + "pass@1": 0.223328125, + "pass@2": 0.26172244094488195, + "pass@4": 0.3050642823397073, + "pass@8": 0.3531824798442372, + "pass@16": 0.40548544167153877, + "pass@32": 0.46207481159213004, + "pass@64": 0.5218905962944446, + "pass@128": 0.581 + }, + "9": { + "pass@1": 0.187609375, + "pass@2": 0.22136023622047235, + "pass@4": 0.25950001049868743, + "pass@8": 0.3039653008171441, + "pass@16": 0.35453158609254953, + "pass@32": 0.40761945730152294, + "pass@64": 0.459340087732421, + "pass@128": 0.508 + }, + "11": { + "pass@1": 0.183015625, + "pass@2": 0.20866338582677174, + "pass@4": 0.23590405080614907, + "pass@8": 0.26657177499164364, + "pass@16": 0.3005953479595448, + "pass@32": 0.3384648822247731, + "pass@64": 0.38032339726143644, + "pass@128": 0.426 + }, + "12": { + "pass@1": 0.178078125, + "pass@2": 0.201375246062992, + "pass@4": 0.22511521456692912, + "pass@8": 0.2512357236143783, + "pass@16": 0.28027664483296916, + "pass@32": 0.31188903107657634, + "pass@64": 0.3440739098252981, + "pass@128": 0.377 + }, + "13": { + "pass@1": 0.13940625, + "pass@2": 0.16364333169291342, + "pass@4": 0.18814229461942247, + "pass@8": 0.21404766874082337, + "pass@16": 0.2409017702074673, + "pass@32": 0.26905766865909836, + "pass@64": 0.3004190790273317, + "pass@128": 0.336 + }, + "14": { + "pass@1": 0.15340625, + "pass@2": 0.1768810285433071, + "pass@4": 0.2000666328271464, + "pass@8": 0.22366324385024125, + "pass@16": 0.24802844539120988, + "pass@32": 0.27423768794616893, + "pass@64": 0.3050598647050578, + "pass@128": 0.342 + }, + "15": { + "pass@1": 0.1334140625, + "pass@2": 0.1554541092519686, + "pass@4": 0.17874117154105718, + "pass@8": 0.20425971936038342, + "pass@16": 0.2329328406948268, + "pass@32": 0.2661451656254728, + "pass@64": 0.30418256883970834, + "pass@128": 0.346 + }, + "16": { + "pass@1": 0.1628203125, + "pass@2": 0.18587364665354322, + "pass@4": 0.207851382358455, + "pass@8": 0.2305969739830245, + "pass@16": 0.25480601080377036, + "pass@32": 0.2811469496260871, + "pass@64": 0.31011607240542965, + "pass@128": 0.342 + }, + "17": { + "pass@1": 0.143125, + "pass@2": 0.16872834645669282, + "pass@4": 0.19324112617172834, + "pass@8": 0.21683406747756903, + "pass@16": 0.24070540026092216, + "pass@32": 0.26551813331097773, + "pass@64": 0.2924959059932723, + "pass@128": 0.324 + }, + "18": { + "pass@1": 0.126140625, + "pass@2": 0.1503442421259842, + "pass@4": 0.17417070163104598, + "pass@8": 0.19968930018568581, + "pass@16": 0.2286068333990339, + "pass@32": 0.2611928633681317, + "pass@64": 0.2964590683968482, + "pass@128": 0.332 + }, + "19": { + "pass@1": 0.135515625, + "pass@2": 0.15826587106299214, + "pass@4": 0.17993976509186338, + "pass@8": 0.20132565635563035, + "pass@16": 0.2238203540907909, + "pass@32": 0.24772720608105048, + "pass@64": 0.27351685907521717, + "pass@128": 0.302 + }, + "20": { + "pass@1": 0.1188046875, + "pass@2": 0.14199384842519686, + "pass@4": 0.16322940363704533, + "pass@8": 0.18380474052090803, + "pass@16": 0.20574088446916178, + "pass@32": 0.23118125948963936, + "pass@64": 0.26222454967024666, + "pass@128": 0.301 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cd07a49c260bd65acab791b10036711b19054d26 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f303317da98d6c99606b3621018edb11050e34126b0ca2e2bb291780f9e47f80 +size 2094263713 diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b3963609bd75d4e3ea8e2d497f75fb1ab609ef84 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab82a4077d45649a20d4fd15bb003c403ec1e2300a3001fe417c70e0ae945f4 +size 400126652 diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl.graph_results.jsonl b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6128cd186ca211773114cc874a4d59e95010b6b0 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19c3dcdc48bf5b01d0069a551c120a36b974365582e747627b8a63c872d114b0 +size 458693098 diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9ac87a5380eda47ffe67b9146297eaeb1aee3e03 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/graphs", "generated_path": "results/context/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2", "results_dir": "results/context/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs", "per_file": [{"file": "results/context/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 102530, "value_mismatch": 442897, "dependency_mismatch": 553756, "answer_mismatch": 691673, "extra_nodes": 762230, "missing_nodes": 1049017, "perfect_with_extra": 411, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "answer_mismatch": 691673, "extra_nodes": 762230, "missing_nodes": 1049017, "value_mismatch": 442897, "dependency_mismatch": 553756, "perfect": 102530, "perfect_with_extra": 411}}, "results_path": "results/context/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/context/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 713905, "dependency_mismatch": 696564, "answer_mismatch": 1091365, "extra_nodes": 974945, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 713905, "dependency_mismatch": 696564, "answer_mismatch": 1091365, "extra_nodes": 974945, "missing_nodes": 1280000}}, "results_path": "results/context/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 102530, "value_mismatch": 1156802, "dependency_mismatch": 1250320, "answer_mismatch": 1783038, "extra_nodes": 1737175, "missing_nodes": 2329017, "perfect_with_extra": 411, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/context/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "answer_mismatch": 1783038, "extra_nodes": 1737175, "missing_nodes": 2329017, "value_mismatch": 1156802, "dependency_mismatch": 1250320, "perfect": 102530, "perfect_with_extra": 411}}}} diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/results.json b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..12186aafe0df8a671ed59014750c899b1a40925d --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.82, + "pass@2": 0.865, + "pass@4": 0.892, + "pass@8": 0.917, + "pass@16": 0.935, + "pass@32": 0.951, + "pass@64": 0.967, + "pass@128": 0.974 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.82, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.865, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.892, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.917, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.935, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.951, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.967, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.974, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.09111111111111111, + "pass@2": 0.0961111111111111, + "pass@4": 0.09911111111111111, + "pass@8": 0.10188888888888889, + "pass@16": 0.10388888888888889, + "pass@32": 0.10566666666666667, + "pass@64": 0.10744444444444444, + "pass@128": 0.10822222222222222 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.82, + "pass@2": 0.865, + "pass@4": 0.892, + "pass@8": 0.917, + "pass@16": 0.935, + "pass@32": 0.951, + "pass@64": 0.967, + "pass@128": 0.974 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.82, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.865, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.892, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.917, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.935, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.951, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.967, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.974, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.09111111111111111, + "pass@2": 0.0961111111111111, + "pass@4": 0.09911111111111111, + "pass@8": 0.10188888888888889, + "pass@16": 0.10388888888888889, + "pass@32": 0.10566666666666667, + "pass@64": 0.10744444444444444, + "pass@128": 0.10822222222222222 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/summary.csv b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c22140d1f887a3f763e3035c5332a421935b900 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm,0.39958767361111114,0.14737265625,0.26684292763157896,0.0,0.0,0.0,140.6659592013889,164.050321875,1152000,1280000,2432000 diff --git a/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/summary.json b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..568b25d0d16cc5cd7f15a2d62a06f639fa948284 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm", + "id_acc": 0.39958767361111114, + "ood_acc": 0.14737265625, + "total_acc": 0.26684292763157896, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 140.6659592013889, + "ood_avg_resp_len": 164.050321875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 648962, + "answer_accuracy": 0.26684292763157896, + "avg_response_len": 152.97351850328948, + "resp_tokens_sum": 372031597, + "pass_at_k": { + "pass@1": 0.26684292763157896, + "pass@2": 0.29722665639245166, + "pass@4": 0.32695936394134406, + "pass@8": 0.357757737288442, + "pass@16": 0.39017345145337295, + "pass@32": 0.42397175746762866, + "pass@64": 0.45869388429861935, + "pass@128": 0.4938421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1830546875, + "pass@2": 0.20952116141732297, + "pass@4": 0.2383966255155605, + "pass@8": 0.2710427312210943, + "pass@16": 0.30699981235999735, + "pass@32": 0.34551654257693415, + "pass@64": 0.38587587679343294, + "pass@128": 0.424 + }, + "2": { + "pass@1": 0.9814609375, + "pass@2": 0.9922114911417323, + "pass@4": 0.9968508753280841, + "pass@8": 0.9988351225906016, + "pass@16": 0.9996941647558666, + "pass@32": 0.9999732028481125, + "pass@64": 0.9999999327042958, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.845484375, + "pass@2": 0.8951834399606313, + "pass@4": 0.9228552554368202, + "pass@8": 0.9404669319809784, + "pass@16": 0.9529833520371419, + "pass@32": 0.961181431816883, + "pass@64": 0.9660464891138485, + "pass@128": 0.969 + }, + "4": { + "pass@1": 0.333078125, + "pass@2": 0.3811968503937009, + "pass@4": 0.4293913296775398, + "pass@8": 0.47646773798502506, + "pass@16": 0.5206730324364229, + "pass@32": 0.5604681683428177, + "pass@64": 0.5957565277950225, + "pass@128": 0.629 + }, + "5": { + "pass@1": 0.29678125, + "pass@2": 0.34330930118110237, + "pass@4": 0.3887239339145104, + "pass@8": 0.43441612237924027, + "pass@16": 0.4809806955064174, + "pass@32": 0.5271614616427763, + "pass@64": 0.5701744077349079, + "pass@128": 0.607 + }, + "6": { + "pass@1": 0.3149921875, + "pass@2": 0.36090834153543305, + "pass@4": 0.40982087579677534, + "pass@8": 0.46266170600441797, + "pass@16": 0.5185506359130283, + "pass@32": 0.5734374683781633, + "pass@64": 0.6231664530144211, + "pass@128": 0.665 + }, + "7": { + "pass@1": 0.2305, + "pass@2": 0.27067015255905474, + "pass@4": 0.3152229831271087, + "pass@8": 0.3643300065773014, + "pass@16": 0.41698232473123015, + "pass@32": 0.4714699999776515, + "pass@64": 0.5240621552910725, + "pass@128": 0.572 + }, + "8": { + "pass@1": 0.223328125, + "pass@2": 0.26172244094488195, + "pass@4": 0.3050642823397073, + "pass@8": 0.3531824798442372, + "pass@16": 0.40548544167153877, + "pass@32": 0.46207481159213004, + "pass@64": 0.5218905962944446, + "pass@128": 0.581 + }, + "9": { + "pass@1": 0.187609375, + "pass@2": 0.22136023622047235, + "pass@4": 0.25950001049868743, + "pass@8": 0.3039653008171441, + "pass@16": 0.35453158609254953, + "pass@32": 0.40761945730152294, + "pass@64": 0.459340087732421, + "pass@128": 0.508 + }, + "11": { + "pass@1": 0.183015625, + "pass@2": 0.20866338582677174, + "pass@4": 0.23590405080614907, + "pass@8": 0.26657177499164364, + "pass@16": 0.3005953479595448, + "pass@32": 0.3384648822247731, + "pass@64": 0.38032339726143644, + "pass@128": 0.426 + }, + "12": { + "pass@1": 0.178078125, + "pass@2": 0.201375246062992, + "pass@4": 0.22511521456692912, + "pass@8": 0.2512357236143783, + "pass@16": 0.28027664483296916, + "pass@32": 0.31188903107657634, + "pass@64": 0.3440739098252981, + "pass@128": 0.377 + }, + "13": { + "pass@1": 0.13940625, + "pass@2": 0.16364333169291342, + "pass@4": 0.18814229461942247, + "pass@8": 0.21404766874082337, + "pass@16": 0.2409017702074673, + "pass@32": 0.26905766865909836, + "pass@64": 0.3004190790273317, + "pass@128": 0.336 + }, + "14": { + "pass@1": 0.15340625, + "pass@2": 0.1768810285433071, + "pass@4": 0.2000666328271464, + "pass@8": 0.22366324385024125, + "pass@16": 0.24802844539120988, + "pass@32": 0.27423768794616893, + "pass@64": 0.3050598647050578, + "pass@128": 0.342 + }, + "15": { + "pass@1": 0.1334140625, + "pass@2": 0.1554541092519686, + "pass@4": 0.17874117154105718, + "pass@8": 0.20425971936038342, + "pass@16": 0.2329328406948268, + "pass@32": 0.2661451656254728, + "pass@64": 0.30418256883970834, + "pass@128": 0.346 + }, + "16": { + "pass@1": 0.1628203125, + "pass@2": 0.18587364665354322, + "pass@4": 0.207851382358455, + "pass@8": 0.2305969739830245, + "pass@16": 0.25480601080377036, + "pass@32": 0.2811469496260871, + "pass@64": 0.31011607240542965, + "pass@128": 0.342 + }, + "17": { + "pass@1": 0.143125, + "pass@2": 0.16872834645669282, + "pass@4": 0.19324112617172834, + "pass@8": 0.21683406747756903, + "pass@16": 0.24070540026092216, + "pass@32": 0.26551813331097773, + "pass@64": 0.2924959059932723, + "pass@128": 0.324 + }, + "18": { + "pass@1": 0.126140625, + "pass@2": 0.1503442421259842, + "pass@4": 0.17417070163104598, + "pass@8": 0.19968930018568581, + "pass@16": 0.2286068333990339, + "pass@32": 0.2611928633681317, + "pass@64": 0.2964590683968482, + "pass@128": 0.332 + }, + "19": { + "pass@1": 0.135515625, + "pass@2": 0.15826587106299214, + "pass@4": 0.17993976509186338, + "pass@8": 0.20132565635563035, + "pass@16": 0.2238203540907909, + "pass@32": 0.24772720608105048, + "pass@64": 0.27351685907521717, + "pass@128": 0.302 + }, + "20": { + "pass@1": 0.1188046875, + "pass@2": 0.14199384842519686, + "pass@4": 0.16322940363704533, + "pass@8": 0.18380474052090803, + "pass@16": 0.20574088446916178, + "pass@32": 0.23118125948963936, + "pass@64": 0.26222454967024666, + "pass@128": 0.301 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9b622ce1153e7e2ec587f23997127b83d015fa8a --- /dev/null +++ b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aca5f59b8480237b0627e469fb7960b56770dd3cb1d63769c282756a1073dc1 +size 1698420071 diff --git a/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_metrics.json b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..864d9d06548eae7af4ee89b3266b62bc712dac09 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm", + "id": { + "answer_accuracy": 0.6095842013888889, + "avg_response_len": 188.3091059027778, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6095842013888889, + "pass@2": 0.6826962762467115, + "pass@4": 0.7425538015352179, + "pass@8": 0.7898305208464987, + "pass@16": 0.8266776034284916, + "pass@32": 0.8554951306538332, + "pass@64": 0.8780938800467579, + "pass@128": 0.8964444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.4126953125, + "pass@2": 0.507600147637796, + "pass@4": 0.5959926069553801, + "pass@8": 0.6732117833496856, + "pass@16": 0.7380794626965851, + "pass@32": 0.7922476964440843, + "pass@64": 0.8373047359451761, + "pass@128": 0.874 + }, + "2": { + "pass@1": 0.9961171875, + "pass@2": 0.9975230068897637, + "pass@4": 0.9983221306711659, + "pass@8": 0.9989727756084528, + "pass@16": 0.9994807207704051, + "pass@32": 0.9998254611556207, + "pass@64": 0.999986175351393, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9103828125, + "pass@2": 0.9425506889763784, + "pass@4": 0.9622388740157479, + "pass@8": 0.975349999295636, + "pass@16": 0.983853120609227, + "pass@32": 0.9890244416004188, + "pass@64": 0.991897819351238, + "pass@128": 0.993 + }, + "4": { + "pass@1": 0.569328125, + "pass@2": 0.6406991879921266, + "pass@4": 0.6936721926321712, + "pass@8": 0.7283366374009416, + "pass@16": 0.7517969322028165, + "pass@32": 0.7706096514107452, + "pass@64": 0.7877617326183003, + "pass@128": 0.804 + }, + "5": { + "pass@1": 0.611265625, + "pass@2": 0.6687064468503937, + "pass@4": 0.7086858456130481, + "pass@8": 0.7409106477256262, + "pass@16": 0.7669154169918215, + "pass@32": 0.7875065182756034, + "pass@64": 0.804226792081769, + "pass@128": 0.818 + }, + "6": { + "pass@1": 0.5552109375, + "pass@2": 0.6553241879921267, + "pass@4": 0.7353291125796779, + "pass@8": 0.7949701863120477, + "pass@16": 0.8385434166582159, + "pass@32": 0.8708625833577998, + "pass@64": 0.8952376000359653, + "pass@128": 0.915 + }, + "7": { + "pass@1": 0.51128125, + "pass@2": 0.6093948080708659, + "pass@4": 0.6908841087364086, + "pass@8": 0.756764869841941, + "pass@16": 0.809663845258288, + "pass@32": 0.8496598066629238, + "pass@64": 0.8777094301364703, + "pass@128": 0.899 + }, + "8": { + "pass@1": 0.494265625, + "pass@2": 0.5934778543307095, + "pass@4": 0.6761844399137602, + "pass@8": 0.7425982996766892, + "pass@16": 0.7955701172611106, + "pass@32": 0.8373145095042487, + "pass@64": 0.8688134871207882, + "pass@128": 0.892 + }, + "9": { + "pass@1": 0.4257109375, + "pass@2": 0.5289901574803154, + "pass@4": 0.621674902699662, + "pass@8": 0.6973594884074764, + "pass@16": 0.7561953984078553, + "pass@32": 0.80240550747302, + "pass@64": 0.8399071477796645, + "pass@128": 0.873 + } + }, + "per_op_accuracy": { + "10": 0.4126953125, + "2": 0.9961171875, + "3": 0.9103828125, + "4": 0.569328125, + "5": 0.611265625, + "6": 0.5552109375, + "7": 0.51128125, + "8": 0.494265625, + "9": 0.4257109375 + }, + "per_op_avg_response_len": { + "10": 253.1405859375, + "2": 108.06421875, + "3": 135.314703125, + "4": 155.6215, + "5": 180.1650078125, + "6": 192.612078125, + "7": 203.909609375, + "8": 225.2549375, + "9": 240.6993125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 253.1405859375, + "2": 108.06421875, + "3": 135.314703125, + "4": 155.6215, + "5": 180.1650078125, + "6": 192.612078125, + "7": 203.909609375, + "8": 225.2549375, + "9": 240.6993125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 702241, + "answer_accuracy": 0.6095842013888889, + "avg_response_len": 188.3091059027778, + "resp_tokens_sum": 216932090, + "pass_at_k": { + "pass@1": 0.6095842013888889, + "pass@2": 0.6826962762467115, + "pass@4": 0.7425538015352179, + "pass@8": 0.7898305208464987, + "pass@16": 0.8266776034284916, + "pass@32": 0.8554951306538332, + "pass@64": 0.8780938800467579, + "pass@128": 0.8964444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.4126953125, + "pass@2": 0.507600147637796, + "pass@4": 0.5959926069553801, + "pass@8": 0.6732117833496856, + "pass@16": 0.7380794626965851, + "pass@32": 0.7922476964440843, + "pass@64": 0.8373047359451761, + "pass@128": 0.874 + }, + "2": { + "pass@1": 0.9961171875, + "pass@2": 0.9975230068897637, + "pass@4": 0.9983221306711659, + "pass@8": 0.9989727756084528, + "pass@16": 0.9994807207704051, + "pass@32": 0.9998254611556207, + "pass@64": 0.999986175351393, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9103828125, + "pass@2": 0.9425506889763784, + "pass@4": 0.9622388740157479, + "pass@8": 0.975349999295636, + "pass@16": 0.983853120609227, + "pass@32": 0.9890244416004188, + "pass@64": 0.991897819351238, + "pass@128": 0.993 + }, + "4": { + "pass@1": 0.569328125, + "pass@2": 0.6406991879921266, + "pass@4": 0.6936721926321712, + "pass@8": 0.7283366374009416, + "pass@16": 0.7517969322028165, + "pass@32": 0.7706096514107452, + "pass@64": 0.7877617326183003, + "pass@128": 0.804 + }, + "5": { + "pass@1": 0.611265625, + "pass@2": 0.6687064468503937, + "pass@4": 0.7086858456130481, + "pass@8": 0.7409106477256262, + "pass@16": 0.7669154169918215, + "pass@32": 0.7875065182756034, + "pass@64": 0.804226792081769, + "pass@128": 0.818 + }, + "6": { + "pass@1": 0.5552109375, + "pass@2": 0.6553241879921267, + "pass@4": 0.7353291125796779, + "pass@8": 0.7949701863120477, + "pass@16": 0.8385434166582159, + "pass@32": 0.8708625833577998, + "pass@64": 0.8952376000359653, + "pass@128": 0.915 + }, + "7": { + "pass@1": 0.51128125, + "pass@2": 0.6093948080708659, + "pass@4": 0.6908841087364086, + "pass@8": 0.756764869841941, + "pass@16": 0.809663845258288, + "pass@32": 0.8496598066629238, + "pass@64": 0.8777094301364703, + "pass@128": 0.899 + }, + "8": { + "pass@1": 0.494265625, + "pass@2": 0.5934778543307095, + "pass@4": 0.6761844399137602, + "pass@8": 0.7425982996766892, + "pass@16": 0.7955701172611106, + "pass@32": 0.8373145095042487, + "pass@64": 0.8688134871207882, + "pass@128": 0.892 + }, + "9": { + "pass@1": 0.4257109375, + "pass@2": 0.5289901574803154, + "pass@4": 0.621674902699662, + "pass@8": 0.6973594884074764, + "pass@16": 0.7561953984078553, + "pass@32": 0.80240550747302, + "pass@64": 0.8399071477796645, + "pass@128": 0.873 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.26865078125, + "avg_response_len": 329.576009375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.26865078125, + "pass@2": 0.35979833907480946, + "pass@4": 0.4505059086895406, + "pass@8": 0.5354268424583083, + "pass@16": 0.6124348192262845, + "pass@32": 0.6816130252627676, + "pass@64": 0.7431988665190546, + "pass@128": 0.7964 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3774140625, + "pass@2": 0.47613705708661475, + "pass@4": 0.5642286361079859, + "pass@8": 0.6398687454955166, + "pass@16": 0.7064828961710622, + "pass@32": 0.7651126517689707, + "pass@64": 0.8155510166375527, + "pass@128": 0.86 + }, + "12": { + "pass@1": 0.3462890625, + "pass@2": 0.43872404035433077, + "pass@4": 0.5239153018372701, + "pass@8": 0.5990279281078111, + "pass@16": 0.66361095472774, + "pass@32": 0.7185271454523737, + "pass@64": 0.767031490951785, + "pass@128": 0.81 + }, + "13": { + "pass@1": 0.3391015625, + "pass@2": 0.4317790354330709, + "pass@4": 0.5181833852643416, + "pass@8": 0.5964838833520059, + "pass@16": 0.6654815946565797, + "pass@32": 0.7258743492382231, + "pass@64": 0.7780189814625351, + "pass@128": 0.823 + }, + "14": { + "pass@1": 0.3178984375, + "pass@2": 0.41741326279527563, + "pass@4": 0.5151516998500183, + "pass@8": 0.6058998559790486, + "pass@16": 0.6868555981134699, + "pass@32": 0.7566070244002328, + "pass@64": 0.8129041237583818, + "pass@128": 0.853 + }, + "15": { + "pass@1": 0.2464140625, + "pass@2": 0.33977239173228363, + "pass@4": 0.4343145795838022, + "pass@8": 0.5257461444108301, + "pass@16": 0.6110246382597375, + "pass@32": 0.6873883168003965, + "pass@64": 0.7533468237790689, + "pass@128": 0.807 + }, + "16": { + "pass@1": 0.303984375, + "pass@2": 0.40927595964566926, + "pass@4": 0.5103989389763774, + "pass@8": 0.6014718394871122, + "pass@16": 0.6813868600977412, + "pass@32": 0.7502524053212755, + "pass@64": 0.8061733769962259, + "pass@128": 0.847 + }, + "17": { + "pass@1": 0.226390625, + "pass@2": 0.31797650098425195, + "pass@4": 0.4140369549118857, + "pass@8": 0.505305026834578, + "pass@16": 0.5872945080924371, + "pass@32": 0.6601433644830466, + "pass@64": 0.7246326001567372, + "pass@128": 0.779 + }, + "18": { + "pass@1": 0.195859375, + "pass@2": 0.2821972194881889, + "pass@4": 0.37402928880764974, + "pass@8": 0.4643392170092771, + "pass@16": 0.5504452676774946, + "pass@32": 0.631118943787985, + "pass@64": 0.7060759494181154, + "pass@128": 0.777 + }, + "19": { + "pass@1": 0.1715546875, + "pass@2": 0.24920460137795278, + "pass@4": 0.33425357292838365, + "pass@8": 0.4190603164372725, + "pass@16": 0.4990036239598838, + "pass@32": 0.5745888217956253, + "pass@64": 0.6477536211565927, + "pass@128": 0.719 + }, + "20": { + "pass@1": 0.1616015625, + "pass@2": 0.23550332185039363, + "pass@4": 0.3165467286276713, + "pass@8": 0.39706546746953736, + "pass@16": 0.47276225050650716, + "pass@32": 0.5465172295795503, + "pass@64": 0.6205006808734961, + "pass@128": 0.689 + } + }, + "per_op_accuracy": { + "11": 0.3774140625, + "12": 0.3462890625, + "13": 0.3391015625, + "14": 0.3178984375, + "15": 0.2464140625, + "16": 0.303984375, + "17": 0.226390625, + "18": 0.195859375, + "19": 0.1715546875, + "20": 0.1616015625 + }, + "per_op_avg_response_len": { + "11": 272.1738515625, + "12": 281.6792734375, + "13": 292.1500078125, + "14": 314.9438046875, + "15": 324.6904453125, + "16": 349.870734375, + "17": 355.5565859375, + "18": 363.7216328125, + "19": 366.1788515625, + "20": 374.79490625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 272.1738515625, + "12": 281.6792734375, + "13": 292.1500078125, + "14": 314.9438046875, + "15": 324.6904453125, + "16": 349.870734375, + "17": 355.5565859375, + "18": 363.7216328125, + "19": 366.1788515625, + "20": 374.79490625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 343873, + "answer_accuracy": 0.26865078125, + "avg_response_len": 329.576009375, + "resp_tokens_sum": 421857292, + "pass_at_k": { + "pass@1": 0.26865078125, + "pass@2": 0.35979833907480946, + "pass@4": 0.4505059086895406, + "pass@8": 0.5354268424583083, + "pass@16": 0.6124348192262845, + "pass@32": 0.6816130252627676, + "pass@64": 0.7431988665190546, + "pass@128": 0.7964 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3774140625, + "pass@2": 0.47613705708661475, + "pass@4": 0.5642286361079859, + "pass@8": 0.6398687454955166, + "pass@16": 0.7064828961710622, + "pass@32": 0.7651126517689707, + "pass@64": 0.8155510166375527, + "pass@128": 0.86 + }, + "12": { + "pass@1": 0.3462890625, + "pass@2": 0.43872404035433077, + "pass@4": 0.5239153018372701, + "pass@8": 0.5990279281078111, + "pass@16": 0.66361095472774, + "pass@32": 0.7185271454523737, + "pass@64": 0.767031490951785, + "pass@128": 0.81 + }, + "13": { + "pass@1": 0.3391015625, + "pass@2": 0.4317790354330709, + "pass@4": 0.5181833852643416, + "pass@8": 0.5964838833520059, + "pass@16": 0.6654815946565797, + "pass@32": 0.7258743492382231, + "pass@64": 0.7780189814625351, + "pass@128": 0.823 + }, + "14": { + "pass@1": 0.3178984375, + "pass@2": 0.41741326279527563, + "pass@4": 0.5151516998500183, + "pass@8": 0.6058998559790486, + "pass@16": 0.6868555981134699, + "pass@32": 0.7566070244002328, + "pass@64": 0.8129041237583818, + "pass@128": 0.853 + }, + "15": { + "pass@1": 0.2464140625, + "pass@2": 0.33977239173228363, + "pass@4": 0.4343145795838022, + "pass@8": 0.5257461444108301, + "pass@16": 0.6110246382597375, + "pass@32": 0.6873883168003965, + "pass@64": 0.7533468237790689, + "pass@128": 0.807 + }, + "16": { + "pass@1": 0.303984375, + "pass@2": 0.40927595964566926, + "pass@4": 0.5103989389763774, + "pass@8": 0.6014718394871122, + "pass@16": 0.6813868600977412, + "pass@32": 0.7502524053212755, + "pass@64": 0.8061733769962259, + "pass@128": 0.847 + }, + "17": { + "pass@1": 0.226390625, + "pass@2": 0.31797650098425195, + "pass@4": 0.4140369549118857, + "pass@8": 0.505305026834578, + "pass@16": 0.5872945080924371, + "pass@32": 0.6601433644830466, + "pass@64": 0.7246326001567372, + "pass@128": 0.779 + }, + "18": { + "pass@1": 0.195859375, + "pass@2": 0.2821972194881889, + "pass@4": 0.37402928880764974, + "pass@8": 0.4643392170092771, + "pass@16": 0.5504452676774946, + "pass@32": 0.631118943787985, + "pass@64": 0.7060759494181154, + "pass@128": 0.777 + }, + "19": { + "pass@1": 0.1715546875, + "pass@2": 0.24920460137795278, + "pass@4": 0.33425357292838365, + "pass@8": 0.4190603164372725, + "pass@16": 0.4990036239598838, + "pass@32": 0.5745888217956253, + "pass@64": 0.6477536211565927, + "pass@128": 0.719 + }, + "20": { + "pass@1": 0.1616015625, + "pass@2": 0.23550332185039363, + "pass@4": 0.3165467286276713, + "pass@8": 0.39706546746953736, + "pass@16": 0.47276225050650716, + "pass@32": 0.5465172295795503, + "pass@64": 0.6205006808734961, + "pass@128": 0.689 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.43014555921052633, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.43014555921052633, + "pass@2": 0.5127499935246315, + "pass@4": 0.5888443842479902, + "pass@8": 0.65593384801061, + "pass@16": 0.7139182433220806, + "pass@32": 0.7639782330796132, + "pass@64": 0.8070965045058616, + "pass@128": 0.8437894736842105 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.4126953125, + "pass@2": 0.507600147637796, + "pass@4": 0.5959926069553801, + "pass@8": 0.6732117833496856, + "pass@16": 0.7380794626965851, + "pass@32": 0.7922476964440843, + "pass@64": 0.8373047359451761, + "pass@128": 0.874 + }, + "2": { + "pass@1": 0.9961171875, + "pass@2": 0.9975230068897637, + "pass@4": 0.9983221306711659, + "pass@8": 0.9989727756084528, + "pass@16": 0.9994807207704051, + "pass@32": 0.9998254611556207, + "pass@64": 0.999986175351393, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9103828125, + "pass@2": 0.9425506889763784, + "pass@4": 0.9622388740157479, + "pass@8": 0.975349999295636, + "pass@16": 0.983853120609227, + "pass@32": 0.9890244416004188, + "pass@64": 0.991897819351238, + "pass@128": 0.993 + }, + "4": { + "pass@1": 0.569328125, + "pass@2": 0.6406991879921266, + "pass@4": 0.6936721926321712, + "pass@8": 0.7283366374009416, + "pass@16": 0.7517969322028165, + "pass@32": 0.7706096514107452, + "pass@64": 0.7877617326183003, + "pass@128": 0.804 + }, + "5": { + "pass@1": 0.611265625, + "pass@2": 0.6687064468503937, + "pass@4": 0.7086858456130481, + "pass@8": 0.7409106477256262, + "pass@16": 0.7669154169918215, + "pass@32": 0.7875065182756034, + "pass@64": 0.804226792081769, + "pass@128": 0.818 + }, + "6": { + "pass@1": 0.5552109375, + "pass@2": 0.6553241879921267, + "pass@4": 0.7353291125796779, + "pass@8": 0.7949701863120477, + "pass@16": 0.8385434166582159, + "pass@32": 0.8708625833577998, + "pass@64": 0.8952376000359653, + "pass@128": 0.915 + }, + "7": { + "pass@1": 0.51128125, + "pass@2": 0.6093948080708659, + "pass@4": 0.6908841087364086, + "pass@8": 0.756764869841941, + "pass@16": 0.809663845258288, + "pass@32": 0.8496598066629238, + "pass@64": 0.8777094301364703, + "pass@128": 0.899 + }, + "8": { + "pass@1": 0.494265625, + "pass@2": 0.5934778543307095, + "pass@4": 0.6761844399137602, + "pass@8": 0.7425982996766892, + "pass@16": 0.7955701172611106, + "pass@32": 0.8373145095042487, + "pass@64": 0.8688134871207882, + "pass@128": 0.892 + }, + "9": { + "pass@1": 0.4257109375, + "pass@2": 0.5289901574803154, + "pass@4": 0.621674902699662, + "pass@8": 0.6973594884074764, + "pass@16": 0.7561953984078553, + "pass@32": 0.80240550747302, + "pass@64": 0.8399071477796645, + "pass@128": 0.873 + }, + "11": { + "pass@1": 0.3774140625, + "pass@2": 0.47613705708661475, + "pass@4": 0.5642286361079859, + "pass@8": 0.6398687454955166, + "pass@16": 0.7064828961710622, + "pass@32": 0.7651126517689707, + "pass@64": 0.8155510166375527, + "pass@128": 0.86 + }, + "12": { + "pass@1": 0.3462890625, + "pass@2": 0.43872404035433077, + "pass@4": 0.5239153018372701, + "pass@8": 0.5990279281078111, + "pass@16": 0.66361095472774, + "pass@32": 0.7185271454523737, + "pass@64": 0.767031490951785, + "pass@128": 0.81 + }, + "13": { + "pass@1": 0.3391015625, + "pass@2": 0.4317790354330709, + "pass@4": 0.5181833852643416, + "pass@8": 0.5964838833520059, + "pass@16": 0.6654815946565797, + "pass@32": 0.7258743492382231, + "pass@64": 0.7780189814625351, + "pass@128": 0.823 + }, + "14": { + "pass@1": 0.3178984375, + "pass@2": 0.41741326279527563, + "pass@4": 0.5151516998500183, + "pass@8": 0.6058998559790486, + "pass@16": 0.6868555981134699, + "pass@32": 0.7566070244002328, + "pass@64": 0.8129041237583818, + "pass@128": 0.853 + }, + "15": { + "pass@1": 0.2464140625, + "pass@2": 0.33977239173228363, + "pass@4": 0.4343145795838022, + "pass@8": 0.5257461444108301, + "pass@16": 0.6110246382597375, + "pass@32": 0.6873883168003965, + "pass@64": 0.7533468237790689, + "pass@128": 0.807 + }, + "16": { + "pass@1": 0.303984375, + "pass@2": 0.40927595964566926, + "pass@4": 0.5103989389763774, + "pass@8": 0.6014718394871122, + "pass@16": 0.6813868600977412, + "pass@32": 0.7502524053212755, + "pass@64": 0.8061733769962259, + "pass@128": 0.847 + }, + "17": { + "pass@1": 0.226390625, + "pass@2": 0.31797650098425195, + "pass@4": 0.4140369549118857, + "pass@8": 0.505305026834578, + "pass@16": 0.5872945080924371, + "pass@32": 0.6601433644830466, + "pass@64": 0.7246326001567372, + "pass@128": 0.779 + }, + "18": { + "pass@1": 0.195859375, + "pass@2": 0.2821972194881889, + "pass@4": 0.37402928880764974, + "pass@8": 0.4643392170092771, + "pass@16": 0.5504452676774946, + "pass@32": 0.631118943787985, + "pass@64": 0.7060759494181154, + "pass@128": 0.777 + }, + "19": { + "pass@1": 0.1715546875, + "pass@2": 0.24920460137795278, + "pass@4": 0.33425357292838365, + "pass@8": 0.4190603164372725, + "pass@16": 0.4990036239598838, + "pass@32": 0.5745888217956253, + "pass@64": 0.6477536211565927, + "pass@128": 0.719 + }, + "20": { + "pass@1": 0.1616015625, + "pass@2": 0.23550332185039363, + "pass@4": 0.3165467286276713, + "pass@8": 0.39706546746953736, + "pass@16": 0.47276225050650716, + "pass@32": 0.5465172295795503, + "pass@64": 0.6205006808734961, + "pass@128": 0.689 + } + }, + "per_op_accuracy": { + "10": 0.4126953125, + "2": 0.9961171875, + "3": 0.9103828125, + "4": 0.569328125, + "5": 0.611265625, + "6": 0.5552109375, + "7": 0.51128125, + "8": 0.494265625, + "9": 0.4257109375, + "11": 0.3774140625, + "12": 0.3462890625, + "13": 0.3391015625, + "14": 0.3178984375, + "15": 0.2464140625, + "16": 0.303984375, + "17": 0.226390625, + "18": 0.195859375, + "19": 0.1715546875, + "20": 0.1616015625 + }, + "per_op_avg_response_len": { + "10": 253.1405859375, + "2": 108.06421875, + "3": 135.314703125, + "4": 155.6215, + "5": 180.1650078125, + "6": 192.612078125, + "7": 203.909609375, + "8": 225.2549375, + "9": 240.6993125, + "11": 272.1738515625, + "12": 281.6792734375, + "13": 292.1500078125, + "14": 314.9438046875, + "15": 324.6904453125, + "16": 349.870734375, + "17": 355.5565859375, + "18": 363.7216328125, + "19": 366.1788515625, + "20": 374.79490625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 253.1405859375, + "2": 108.06421875, + "3": 135.314703125, + "4": 155.6215, + "5": 180.1650078125, + "6": 192.612078125, + "7": 203.909609375, + "8": 225.2549375, + "9": 240.6993125, + "11": 272.1738515625, + "12": 281.6792734375, + "13": 292.1500078125, + "14": 314.9438046875, + "15": 324.6904453125, + "16": 349.870734375, + "17": 355.5565859375, + "18": 363.7216328125, + "19": 366.1788515625, + "20": 374.79490625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 1046114, + "answer_accuracy": 0.43014555921052633, + "avg_response_len": 262.6601077302632, + "resp_tokens_sum": 638789382, + "pass_at_k": { + "pass@1": 0.43014555921052633, + "pass@2": 0.5127499935246315, + "pass@4": 0.5888443842479902, + "pass@8": 0.65593384801061, + "pass@16": 0.7139182433220806, + "pass@32": 0.7639782330796132, + "pass@64": 0.8070965045058616, + "pass@128": 0.8437894736842105 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.4126953125, + "pass@2": 0.507600147637796, + "pass@4": 0.5959926069553801, + "pass@8": 0.6732117833496856, + "pass@16": 0.7380794626965851, + "pass@32": 0.7922476964440843, + "pass@64": 0.8373047359451761, + "pass@128": 0.874 + }, + "2": { + "pass@1": 0.9961171875, + "pass@2": 0.9975230068897637, + "pass@4": 0.9983221306711659, + "pass@8": 0.9989727756084528, + "pass@16": 0.9994807207704051, + "pass@32": 0.9998254611556207, + "pass@64": 0.999986175351393, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9103828125, + "pass@2": 0.9425506889763784, + "pass@4": 0.9622388740157479, + "pass@8": 0.975349999295636, + "pass@16": 0.983853120609227, + "pass@32": 0.9890244416004188, + "pass@64": 0.991897819351238, + "pass@128": 0.993 + }, + "4": { + "pass@1": 0.569328125, + "pass@2": 0.6406991879921266, + "pass@4": 0.6936721926321712, + "pass@8": 0.7283366374009416, + "pass@16": 0.7517969322028165, + "pass@32": 0.7706096514107452, + "pass@64": 0.7877617326183003, + "pass@128": 0.804 + }, + "5": { + "pass@1": 0.611265625, + "pass@2": 0.6687064468503937, + "pass@4": 0.7086858456130481, + "pass@8": 0.7409106477256262, + "pass@16": 0.7669154169918215, + "pass@32": 0.7875065182756034, + "pass@64": 0.804226792081769, + "pass@128": 0.818 + }, + "6": { + "pass@1": 0.5552109375, + "pass@2": 0.6553241879921267, + "pass@4": 0.7353291125796779, + "pass@8": 0.7949701863120477, + "pass@16": 0.8385434166582159, + "pass@32": 0.8708625833577998, + "pass@64": 0.8952376000359653, + "pass@128": 0.915 + }, + "7": { + "pass@1": 0.51128125, + "pass@2": 0.6093948080708659, + "pass@4": 0.6908841087364086, + "pass@8": 0.756764869841941, + "pass@16": 0.809663845258288, + "pass@32": 0.8496598066629238, + "pass@64": 0.8777094301364703, + "pass@128": 0.899 + }, + "8": { + "pass@1": 0.494265625, + "pass@2": 0.5934778543307095, + "pass@4": 0.6761844399137602, + "pass@8": 0.7425982996766892, + "pass@16": 0.7955701172611106, + "pass@32": 0.8373145095042487, + "pass@64": 0.8688134871207882, + "pass@128": 0.892 + }, + "9": { + "pass@1": 0.4257109375, + "pass@2": 0.5289901574803154, + "pass@4": 0.621674902699662, + "pass@8": 0.6973594884074764, + "pass@16": 0.7561953984078553, + "pass@32": 0.80240550747302, + "pass@64": 0.8399071477796645, + "pass@128": 0.873 + }, + "11": { + "pass@1": 0.3774140625, + "pass@2": 0.47613705708661475, + "pass@4": 0.5642286361079859, + "pass@8": 0.6398687454955166, + "pass@16": 0.7064828961710622, + "pass@32": 0.7651126517689707, + "pass@64": 0.8155510166375527, + "pass@128": 0.86 + }, + "12": { + "pass@1": 0.3462890625, + "pass@2": 0.43872404035433077, + "pass@4": 0.5239153018372701, + "pass@8": 0.5990279281078111, + "pass@16": 0.66361095472774, + "pass@32": 0.7185271454523737, + "pass@64": 0.767031490951785, + "pass@128": 0.81 + }, + "13": { + "pass@1": 0.3391015625, + "pass@2": 0.4317790354330709, + "pass@4": 0.5181833852643416, + "pass@8": 0.5964838833520059, + "pass@16": 0.6654815946565797, + "pass@32": 0.7258743492382231, + "pass@64": 0.7780189814625351, + "pass@128": 0.823 + }, + "14": { + "pass@1": 0.3178984375, + "pass@2": 0.41741326279527563, + "pass@4": 0.5151516998500183, + "pass@8": 0.6058998559790486, + "pass@16": 0.6868555981134699, + "pass@32": 0.7566070244002328, + "pass@64": 0.8129041237583818, + "pass@128": 0.853 + }, + "15": { + "pass@1": 0.2464140625, + "pass@2": 0.33977239173228363, + "pass@4": 0.4343145795838022, + "pass@8": 0.5257461444108301, + "pass@16": 0.6110246382597375, + "pass@32": 0.6873883168003965, + "pass@64": 0.7533468237790689, + "pass@128": 0.807 + }, + "16": { + "pass@1": 0.303984375, + "pass@2": 0.40927595964566926, + "pass@4": 0.5103989389763774, + "pass@8": 0.6014718394871122, + "pass@16": 0.6813868600977412, + "pass@32": 0.7502524053212755, + "pass@64": 0.8061733769962259, + "pass@128": 0.847 + }, + "17": { + "pass@1": 0.226390625, + "pass@2": 0.31797650098425195, + "pass@4": 0.4140369549118857, + "pass@8": 0.505305026834578, + "pass@16": 0.5872945080924371, + "pass@32": 0.6601433644830466, + "pass@64": 0.7246326001567372, + "pass@128": 0.779 + }, + "18": { + "pass@1": 0.195859375, + "pass@2": 0.2821972194881889, + "pass@4": 0.37402928880764974, + "pass@8": 0.4643392170092771, + "pass@16": 0.5504452676774946, + "pass@32": 0.631118943787985, + "pass@64": 0.7060759494181154, + "pass@128": 0.777 + }, + "19": { + "pass@1": 0.1715546875, + "pass@2": 0.24920460137795278, + "pass@4": 0.33425357292838365, + "pass@8": 0.4190603164372725, + "pass@16": 0.4990036239598838, + "pass@32": 0.5745888217956253, + "pass@64": 0.6477536211565927, + "pass@128": 0.719 + }, + "20": { + "pass@1": 0.1616015625, + "pass@2": 0.23550332185039363, + "pass@4": 0.3165467286276713, + "pass@8": 0.39706546746953736, + "pass@16": 0.47276225050650716, + "pass@32": 0.5465172295795503, + "pass@64": 0.6205006808734961, + "pass@128": 0.689 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_ood_generations.jsonl b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_ood_generations.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3e71840facbe71441bf819971536c92ae2e75aa0 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_ood_generations.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:225d22193d809a8709726abeeb3871417d0be50f27a803d1841af7dbe2e1a73d +size 2563476707 diff --git a/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/graphs/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/graphs/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d67662da6a7146741826cb275b999589e4304dd3 --- /dev/null +++ b/results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/graphs/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7124a81fcbd0bcc3b05a8a6837a53655809004cc6244046681eebf0587a83b1d +size 391112662