{
  "games": {
    "wordle": {
      "n": 393,
      "solved": 37,
      "accuracy": 0.09414758269720101,
      "ci_lo": 0.06907606180854267,
      "ci_hi": 0.1270767608794013,
      "avg_rounds_to_win": 4.081081081081081,
      "solved_by_round": {
        "2": 3,
        "3": 8,
        "4": 12,
        "5": 11,
        "6": 3
      },
      "action_parse_rate": 0.8889374726716223,
      "think_rate": 1.0,
      "good_status": "won",
      "game_no": 0
    },
    "charcount": {
      "n": 300,
      "solved": 0,
      "accuracy": 0.0,
      "ci_lo": 0.0,
      "ci_hi": 0.012643429997735661,
      "avg_rounds_to_win": null,
      "solved_by_round": {},
      "action_parse_rate": 0.61,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 1
    },
    "validity": {
      "n": 300,
      "solved": 45,
      "accuracy": 0.15,
      "ci_lo": 0.1140318876347062,
      "ci_hi": 0.19481851336370873,
      "avg_rounds_to_win": 1.0,
      "solved_by_round": {
        "1": 45
      },
      "action_parse_rate": 0.5433333333333333,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 2
    },
    "anagram": {
      "n": 300,
      "solved": 66,
      "accuracy": 0.22,
      "ci_lo": 0.17682672204095912,
      "ci_hi": 0.2702535987577729,
      "avg_rounds_to_win": 1.0,
      "solved_by_round": {
        "1": 66
      },
      "action_parse_rate": 0.7533333333333333,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 3
    },
    "endstart": {
      "n": 300,
      "solved": 5,
      "accuracy": 0.016666666666666666,
      "ci_lo": 0.0071393720427721935,
      "ci_hi": 0.038415943621705605,
      "avg_rounds_to_win": 1.0,
      "solved_by_round": {
        "1": 5
      },
      "action_parse_rate": 0.9366666666666666,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 4
    },
    "rhyme": {
      "n": 300,
      "solved": 7,
      "accuracy": 0.023333333333333334,
      "ci_lo": 0.01134751559829571,
      "ci_hi": 0.04737255433287896,
      "avg_rounds_to_win": 1.0,
      "solved_by_round": {
        "1": 7
      },
      "action_parse_rate": 0.9633333333333334,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 5
    },
    "crossword": {
      "n": 300,
      "solved": 5,
      "accuracy": 0.016666666666666666,
      "ci_lo": 0.0071393720427721935,
      "ci_hi": 0.038415943621705605,
      "avg_rounds_to_win": 1.0,
      "solved_by_round": {
        "1": 5
      },
      "action_parse_rate": 0.8833333333333333,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 6
    },
    "charset": {
      "n": 300,
      "solved": 0,
      "accuracy": 0.0,
      "ci_lo": 0.0,
      "ci_hi": 0.012643429997735661,
      "avg_rounds_to_win": null,
      "solved_by_round": {},
      "action_parse_rate": 0.51,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 7
    },
    "mistakeid": {
      "n": 300,
      "solved": 0,
      "accuracy": 0.0,
      "ci_lo": 0.0,
      "ci_hi": 0.012643429997735661,
      "avg_rounds_to_win": null,
      "solved_by_round": {},
      "action_parse_rate": 0.4166666666666667,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 8
    },
    "tower": {
      "n": 300,
      "solved": 0,
      "accuracy": 0.0,
      "ci_lo": 0.0,
      "ci_hi": 0.012643429997735661,
      "avg_rounds_to_win": null,
      "solved_by_round": {},
      "action_parse_rate": 0.48,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 9
    },
    "codebreaker": {
      "n": 300,
      "solved": 0,
      "accuracy": 0.0,
      "ci_lo": 0.0,
      "ci_hi": 0.012643429997735661,
      "avg_rounds_to_win": null,
      "solved_by_round": {},
      "action_parse_rate": 0.8066666666666666,
      "think_rate": 1.0,
      "good_status": "won",
      "game_no": 10
    },
    "bullscows": {
      "n": 300,
      "solved": 0,
      "accuracy": 0.0,
      "ci_lo": 0.0,
      "ci_hi": 0.012643429997735661,
      "avg_rounds_to_win": null,
      "solved_by_round": {},
      "action_parse_rate": 0.01,
      "think_rate": 1.0,
      "good_status": "won",
      "game_no": 11
    },
    "consistency": {
      "n": 300,
      "solved": 41,
      "accuracy": 0.13666666666666666,
      "ci_lo": 0.10236459725248649,
      "ci_hi": 0.18015629521253473,
      "avg_rounds_to_win": 1.0,
      "solved_by_round": {
        "1": 41
      },
      "action_parse_rate": 0.93,
      "think_rate": 1.0,
      "good_status": "correct",
      "game_no": 12
    }
  },
  "aggregate": {
    "macro_accuracy": 0.05057545507927187,
    "single_turn_acc": 0.056333333333333326,
    "multi_turn_acc": 0.03138252756573367,
    "reasoning_acc": 0.08270356234096692
  },
  "label": "wordle-e2",
  "model": "saketh-chervu/word-games-sft-wordle",
  "revision": "epoch-2",
  "n": 300,
  "seed": 0,
  "sampling": {
    "temperature": 0.6,
    "top_p": 0.95,
    "enable_thinking": true,
    "max_tokens": 4096
  },
  "raw_dir": "/workspace/eval_results_v2/raw/wordle-e2"
}