{
  "train": {
    "readout": {
      "categorical": "175/246",
      "categorical_accuracy": 0.7113821138211383,
      "completion_found": "45/48",
      "false_completions": "1/96",
      "unknown": "0/48",
      "score_mae": 0.39696706232747864
    },
    "verify_zero_shot": {
      "categorical": "177/246",
      "categorical_accuracy": 0.7195121951219512,
      "completion_found": "47/48",
      "false_completions": "0/96",
      "unknown": "0/48",
      "score_mae": 0.37552783307934967
    },
    "seconds_per_request": {
      "mean": 1.3231873131460614,
      "p95": 1.7301512956619263
    }
  },
  "validation": {
    "readout": {
      "categorical": "92/126",
      "categorical_accuracy": 0.7301587301587301,
      "completion_found": "23/24",
      "false_completions": "0/48",
      "unknown": "2/24",
      "score_mae": 0.3990468073946734
    },
    "verify_zero_shot": {
      "categorical": "89/126",
      "categorical_accuracy": 0.7063492063492064,
      "completion_found": "21/24",
      "false_completions": "0/48",
      "unknown": "1/24",
      "score_mae": 0.28145276563851634
    },
    "seconds_per_request": {
      "mean": 1.2563178936640422,
      "p95": 1.640113389492035
    }
  },
  "test": {
    "readout": {
      "categorical": "86/126",
      "categorical_accuracy": 0.6825396825396826,
      "completion_found": "24/24",
      "false_completions": "0/48",
      "unknown": "0/24",
      "score_mae": 0.5369042108585644
    },
    "verify_zero_shot": {
      "categorical": "85/126",
      "categorical_accuracy": 0.6746031746031746,
      "completion_found": "22/24",
      "false_completions": "0/48",
      "unknown": "0/24",
      "score_mae": 0.42307032748062895
    },
    "seconds_per_request": {
      "mean": 1.279320968521966,
      "p95": 1.6923456430435182
    }
  },
  "total_minutes": 3.8
}