| { | |
| "ckpt": "/scratch/Max/ckpts/sft_lora_v2/best", | |
| "val": "/scratch/Max/data/v2/internal_val_v2_shuf.jsonl", | |
| "val_format": "internal", | |
| "n_items": 1244, | |
| "pass_at_1": 0.5811897106109325, | |
| "correct": 723, | |
| "no_box": 173, | |
| "no_box_pct": 13.906752411575562, | |
| "by_source": { | |
| "gpqa_diamond": { | |
| "correct": 30, | |
| "total": 198, | |
| "no_box": 122 | |
| }, | |
| "arc_challenge_validation": { | |
| "correct": 252, | |
| "total": 299, | |
| "no_box": 7 | |
| }, | |
| "mmlu_pro_validation": { | |
| "correct": 36, | |
| "total": 70, | |
| "no_box": 13 | |
| }, | |
| "mmlu_validation_sub": { | |
| "correct": 309, | |
| "total": 456, | |
| "no_box": 18 | |
| }, | |
| "medmcqa_validation_sub": { | |
| "correct": 96, | |
| "total": 221, | |
| "no_box": 13 | |
| } | |
| }, | |
| "by_n_options": { | |
| "4": { | |
| "correct": 683, | |
| "total": 1170 | |
| }, | |
| "10": { | |
| "correct": 36, | |
| "total": 70 | |
| }, | |
| "3": { | |
| "correct": 3, | |
| "total": 3 | |
| }, | |
| "5": { | |
| "correct": 1, | |
| "total": 1 | |
| } | |
| }, | |
| "gen_tokens_mean": 949.1069131832797, | |
| "gen_tokens_max": 4096, | |
| "max_new_tokens": 4096, | |
| "elapsed_s": 12861.068673849106 | |
| } |