| { |
| "games": { |
| "wordle": { |
| "n": 300, |
| "solved": 7, |
| "accuracy": 0.023333333333333334, |
| "ci_lo": 0.01134751559829571, |
| "ci_hi": 0.04737255433287896, |
| "avg_rounds_to_win": 4.714285714285714, |
| "solved_by_round": { |
| "4": 4, |
| "5": 1, |
| "6": 2 |
| }, |
| "action_parse_rate": 0.7895030709101061, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 0 |
| }, |
| "charcount": { |
| "n": 300, |
| "solved": 1, |
| "accuracy": 0.0033333333333333335, |
| "ci_lo": 0.0005886425761407379, |
| "ci_hi": 0.018637164554943353, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 1 |
| }, |
| "action_parse_rate": 0.72, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 1 |
| }, |
| "validity": { |
| "n": 300, |
| "solved": 39, |
| "accuracy": 0.13, |
| "ci_lo": 0.09657483743516235, |
| "ci_hi": 0.17278130076316203, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 39 |
| }, |
| "action_parse_rate": 0.5033333333333333, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 2 |
| }, |
| "anagram": { |
| "n": 300, |
| "solved": 71, |
| "accuracy": 0.23666666666666666, |
| "ci_lo": 0.19208795133542217, |
| "ci_hi": 0.28790425513005197, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 71 |
| }, |
| "action_parse_rate": 0.77, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 3 |
| }, |
| "endstart": { |
| "n": 300, |
| "solved": 11, |
| "accuracy": 0.03666666666666667, |
| "ci_lo": 0.02059509146818067, |
| "ci_hi": 0.06445448699638771, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 11 |
| }, |
| "action_parse_rate": 0.9466666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 4 |
| }, |
| "rhyme": { |
| "n": 300, |
| "solved": 7, |
| "accuracy": 0.023333333333333334, |
| "ci_lo": 0.01134751559829571, |
| "ci_hi": 0.04737255433287896, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 7 |
| }, |
| "action_parse_rate": 0.9633333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 5 |
| }, |
| "crossword": { |
| "n": 300, |
| "solved": 1, |
| "accuracy": 0.0033333333333333335, |
| "ci_lo": 0.0005886425761407379, |
| "ci_hi": 0.018637164554943353, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 1 |
| }, |
| "action_parse_rate": 0.78, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 6 |
| }, |
| "charset": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.5933333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 7 |
| }, |
| "mistakeid": { |
| "n": 300, |
| "solved": 4, |
| "accuracy": 0.013333333333333334, |
| "ci_lo": 0.005196877684679467, |
| "ci_hi": 0.03377606084644991, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 4 |
| }, |
| "action_parse_rate": 0.5166666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 8 |
| }, |
| "tower": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.56, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 9 |
| }, |
| "codebreaker": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.76, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 10 |
| }, |
| "bullscows": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.006666666666666667, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 11 |
| }, |
| "consistency": { |
| "n": 300, |
| "solved": 70, |
| "accuracy": 0.23333333333333334, |
| "ci_lo": 0.18902749123816998, |
| "ci_hi": 0.28438233809395574, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 70 |
| }, |
| "action_parse_rate": 0.8433333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 12 |
| } |
| }, |
| "aggregate": { |
| "macro_accuracy": 0.0541025641025641, |
| "single_turn_acc": 0.06799999999999999, |
| "multi_turn_acc": 0.007777777777777778, |
| "reasoning_acc": 0.06916666666666667 |
| }, |
| "label": "wordle-e1", |
| "model": "saketh-chervu/word-games-sft-wordle", |
| "revision": "epoch-1", |
| "n": 300, |
| "seed": 0, |
| "sampling": { |
| "temperature": 0.6, |
| "top_p": 0.95, |
| "enable_thinking": true, |
| "max_tokens": 4096 |
| }, |
| "raw_dir": "/workspace/eval_results_v2/raw/wordle-e1" |
| } |
