| { |
| "games": { |
| "wordle": { |
| "n": 393, |
| "solved": 37, |
| "accuracy": 0.09414758269720101, |
| "ci_lo": 0.06907606180854267, |
| "ci_hi": 0.1270767608794013, |
| "avg_rounds_to_win": 4.081081081081081, |
| "solved_by_round": { |
| "2": 3, |
| "3": 8, |
| "4": 12, |
| "5": 11, |
| "6": 3 |
| }, |
| "action_parse_rate": 0.8889374726716223, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 0 |
| }, |
| "charcount": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.61, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 1 |
| }, |
| "validity": { |
| "n": 300, |
| "solved": 45, |
| "accuracy": 0.15, |
| "ci_lo": 0.1140318876347062, |
| "ci_hi": 0.19481851336370873, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 45 |
| }, |
| "action_parse_rate": 0.5433333333333333, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 2 |
| }, |
| "anagram": { |
| "n": 300, |
| "solved": 66, |
| "accuracy": 0.22, |
| "ci_lo": 0.17682672204095912, |
| "ci_hi": 0.2702535987577729, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 66 |
| }, |
| "action_parse_rate": 0.7533333333333333, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 3 |
| }, |
| "endstart": { |
| "n": 300, |
| "solved": 5, |
| "accuracy": 0.016666666666666666, |
| "ci_lo": 0.0071393720427721935, |
| "ci_hi": 0.038415943621705605, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 5 |
| }, |
| "action_parse_rate": 0.9366666666666666, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 4 |
| }, |
| "rhyme": { |
| "n": 300, |
| "solved": 7, |
| "accuracy": 0.023333333333333334, |
| "ci_lo": 0.01134751559829571, |
| "ci_hi": 0.04737255433287896, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 7 |
| }, |
| "action_parse_rate": 0.9633333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 5 |
| }, |
| "crossword": { |
| "n": 300, |
| "solved": 5, |
| "accuracy": 0.016666666666666666, |
| "ci_lo": 0.0071393720427721935, |
| "ci_hi": 0.038415943621705605, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 5 |
| }, |
| "action_parse_rate": 0.8833333333333333, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 6 |
| }, |
| "charset": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.51, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 7 |
| }, |
| "mistakeid": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.4166666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 8 |
| }, |
| "tower": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.48, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 9 |
| }, |
| "codebreaker": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.8066666666666666, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 10 |
| }, |
| "bullscows": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.01, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 11 |
| }, |
| "consistency": { |
| "n": 300, |
| "solved": 41, |
| "accuracy": 0.13666666666666666, |
| "ci_lo": 0.10236459725248649, |
| "ci_hi": 0.18015629521253473, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 41 |
| }, |
| "action_parse_rate": 0.93, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 12 |
| } |
| }, |
| "aggregate": { |
| "macro_accuracy": 0.05057545507927187, |
| "single_turn_acc": 0.056333333333333326, |
| "multi_turn_acc": 0.03138252756573367, |
| "reasoning_acc": 0.08270356234096692 |
| }, |
| "label": "wordle-e2", |
| "model": "saketh-chervu/word-games-sft-wordle", |
| "revision": "epoch-2", |
| "n": 300, |
| "seed": 0, |
| "sampling": { |
| "temperature": 0.6, |
| "top_p": 0.95, |
| "enable_thinking": true, |
| "max_tokens": 4096 |
| }, |
| "raw_dir": "/workspace/eval_results_v2/raw/wordle-e2" |
| } |
