| { |
| "games": { |
| "wordle": { |
| "n": 300, |
| "solved": 22, |
| "accuracy": 0.07333333333333333, |
| "ci_lo": 0.048923631132474436, |
| "ci_hi": 0.10853209579892667, |
| "avg_rounds_to_win": 4.409090909090909, |
| "solved_by_round": { |
| "2": 2, |
| "3": 4, |
| "4": 4, |
| "5": 7, |
| "6": 5 |
| }, |
| "action_parse_rate": 0.826628895184136, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 0 |
| }, |
| "charcount": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.67, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 1 |
| }, |
| "validity": { |
| "n": 300, |
| "solved": 21, |
| "accuracy": 0.07, |
| "ci_lo": 0.04623659396060609, |
| "ci_hi": 0.10463675583744661, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 21 |
| }, |
| "action_parse_rate": 0.3566666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 2 |
| }, |
| "anagram": { |
| "n": 300, |
| "solved": 47, |
| "accuracy": 0.15666666666666668, |
| "ci_lo": 0.11990622028482688, |
| "ci_hi": 0.20210893498028493, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 47 |
| }, |
| "action_parse_rate": 0.63, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 3 |
| }, |
| "endstart": { |
| "n": 300, |
| "solved": 10, |
| "accuracy": 0.03333333333333333, |
| "ci_lo": 0.018204749873650852, |
| "ci_hi": 0.0602624514575691, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 10 |
| }, |
| "action_parse_rate": 0.9266666666666666, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 4 |
| }, |
| "rhyme": { |
| "n": 300, |
| "solved": 3, |
| "accuracy": 0.01, |
| "ci_lo": 0.0034065567627181124, |
| "ci_hi": 0.028984004635062833, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 3 |
| }, |
| "action_parse_rate": 0.9633333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 5 |
| }, |
| "crossword": { |
| "n": 300, |
| "solved": 3, |
| "accuracy": 0.01, |
| "ci_lo": 0.0034065567627181124, |
| "ci_hi": 0.028984004635062833, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 3 |
| }, |
| "action_parse_rate": 0.7133333333333334, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 6 |
| }, |
| "charset": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.5166666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 7 |
| }, |
| "mistakeid": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.6066666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 8 |
| }, |
| "tower": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.54, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 9 |
| }, |
| "codebreaker": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.7241666666666666, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 10 |
| }, |
| "bullscows": { |
| "n": 300, |
| "solved": 0, |
| "accuracy": 0.0, |
| "ci_lo": 0.0, |
| "ci_hi": 0.012643429997735661, |
| "avg_rounds_to_win": null, |
| "solved_by_round": {}, |
| "action_parse_rate": 0.0026666666666666666, |
| "think_rate": 1.0, |
| "good_status": "won", |
| "game_no": 11 |
| }, |
| "consistency": { |
| "n": 300, |
| "solved": 28, |
| "accuracy": 0.09333333333333334, |
| "ci_lo": 0.06536379092760147, |
| "ci_hi": 0.13158619880389022, |
| "avg_rounds_to_win": 1.0, |
| "solved_by_round": { |
| "1": 28 |
| }, |
| "action_parse_rate": 0.8366666666666667, |
| "think_rate": 1.0, |
| "good_status": "correct", |
| "game_no": 12 |
| } |
| }, |
| "aggregate": { |
| "macro_accuracy": 0.034358974358974365, |
| "single_turn_acc": 0.037333333333333336, |
| "multi_turn_acc": 0.024444444444444446, |
| "reasoning_acc": 0.060000000000000005 |
| }, |
| "label": "wordle-e3", |
| "model": "saketh-chervu/word-games-sft-wordle", |
| "revision": "epoch-3", |
| "n": 300, |
| "seed": 0, |
| "sampling": { |
| "temperature": 0.6, |
| "top_p": 0.95, |
| "enable_thinking": true, |
| "max_tokens": 4096 |
| }, |
| "raw_dir": "/workspace/eval_results_v2/raw/wordle-e3" |
| } |
