{
  "ckpt": "/scratch/Max/ckpts/sft_lora_v2/best",
  "val": "/scratch/Max/data/v2/internal_val_v2_shuf.jsonl",
  "val_format": "internal",
  "n_items": 1244,
  "pass_at_1": 0.5811897106109325,
  "correct": 723,
  "no_box": 173,
  "no_box_pct": 13.906752411575562,
  "by_source": {
    "gpqa_diamond": {
      "correct": 30,
      "total": 198,
      "no_box": 122
    },
    "arc_challenge_validation": {
      "correct": 252,
      "total": 299,
      "no_box": 7
    },
    "mmlu_pro_validation": {
      "correct": 36,
      "total": 70,
      "no_box": 13
    },
    "mmlu_validation_sub": {
      "correct": 309,
      "total": 456,
      "no_box": 18
    },
    "medmcqa_validation_sub": {
      "correct": 96,
      "total": 221,
      "no_box": 13
    }
  },
  "by_n_options": {
    "4": {
      "correct": 683,
      "total": 1170
    },
    "10": {
      "correct": 36,
      "total": 70
    },
    "3": {
      "correct": 3,
      "total": 3
    },
    "5": {
      "correct": 1,
      "total": 1
    }
  },
  "gen_tokens_mean": 949.1069131832797,
  "gen_tokens_max": 4096,
  "max_new_tokens": 4096,
  "elapsed_s": 12861.068673849106
}