| { |
| "games": { |
| "wordle": { |
| "n": 300, |
| "solved": 20, |
| "accuracy": 0.06666666666666667, |
| "ci_lo": 0.04356717679698387, |
| "ci_hi": 0.10072379586772039, |
| "avg_rounds_to_win": 4.65, |
| "solved_by_round": { |
| "3": 3, |
| "4": 7, |
| "5": 4, |
| "6": 6 |
| }, |
| "action_parse_rate": 0.7783417935702199, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 0 |
| }, |
| "charcount": { |
| "n": 300, |
| "solved": 1, |
| "accuracy": 0.0033333333333333335, |
| "ci_lo": 0.0005886425761407379, |
| "ci_hi": 0.018637164554943353, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 1 |
| }, |
| "action_parse_rate": 0.6333333333333333, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 1 |
| }, |
| "validity": { |
| "n": 300, |
| "solved": 15, |
| "accuracy": 0.05, |
| "ci_lo": 0.030531367290852585, |
| "ci_hi": 0.08084771970710952, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 15 |
| }, |
| "action_parse_rate": 0.36333333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 2 |
| }, |
| "anagram": { |
| "n": 300, |
| "solved": 30, |
| "accuracy": 0.1, |
| "ci_lo": 0.07094746564722294, |
| "ci_hi": 0.1391672783509656, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 30 |
| }, |
| "action_parse_rate": 0.6033333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 3 |
| }, |
| "endstart": { |
| "n": 300, |
| "solved": 12, |
| "accuracy": 0.04, |
| "ci_lo": 0.02302709213793877, |
| "ci_hi": 0.06860486345997804, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 12 |
| }, |
| "action_parse_rate": 0.9166666666666666, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 4 |
| }, |
| "rhyme": { |
| "n": 300, |
| "solved": 7, |
| "accuracy": 0.023333333333333334, |
| "ci_lo": 0.01134751559829571, |
| "ci_hi": 0.04737255433287896, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 7 |
| }, |
| "action_parse_rate": 0.9666666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 5 |
| }, |
| "crossword": { |
| "n": 300, |
| "solved": 3, |
| "accuracy": 0.01, |
| "ci_lo": 0.0034065567627181124, |
| "ci_hi": 0.028984004635062833, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 3 |
| }, |
| "action_parse_rate": 0.73, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 6 |
| }, |
| "charset": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.4866666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 7 |
| }, |
| "mistakeid": { |
| "n": 300, |
| "solved": 1, |
| "accuracy": 0.0033333333333333335, |
| "ci_lo": 0.0005886425761407379, |
| "ci_hi": 0.018637164554943353, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 1 |
| }, |
| "action_parse_rate": 0.56, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 8 |
| }, |
| "tower": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.5566666666666666, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 9 |
| }, |
| "codebreaker": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.6455555555555555, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 10 |
| }, |
| "bullscows": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.006666666666666667, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 11 |
| }, |
| "consistency": { |
| "n": 300, |
| "solved": 42, |
| "accuracy": 0.14, |
| "ci_lo": 0.1052707698510638, |
| "ci_hi": 0.18383249974730587, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 42 |
| }, |
| "action_parse_rate": 0.8466666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 12 |
| } |
| }, |
| "aggregate": { |
| "macro_accuracy": 0.03358974358974359, |
| "single_turn_acc": 0.037000000000000005, |
| "multi_turn_acc": 0.022222222222222223, |
| "reasoning_acc": 0.045000000000000005 |
| }, |
| "label": "wordle-e4", |
| "model": "saketh-chervu/word-games-sft-wordle", |
| "revision": "epoch-4", |
| "n": 300, |
| "seed": 0, |
| "sampling": { |
| "temperature": 0.6, |
| "top_p": 0.95, |
| "enable_thinking": true, |
| "max_tokens": 4096 |
| }, |
| "raw_dir": "/workspace/eval_results_v2/raw/wordle-e4" |
| } |
