Upload eval/think_eval_116000_gsm_192.json with huggingface_hub

eval/think_eval_116000_gsm_192.json

@@ -0,0 +1,399 @@
+{
+  "checkpoint": "outputs/molmo-audio-lora-diar-d-v35/checkpoint-116000",
+  "scaffold": true,
+  "n_p1_close": 20,
+  "n_cued": 20,
+  "n_form": 20,
+  "n_exact": 8,
+  "n_unearned": 0,
+  "uncued_leak": 0,
+  "median_tokens": 14.0,
+  "gates": {
+    "cued_format": true,
+    "cued_exact": false,
+    "uncued_leak": true,
+    "unearned_correct": true,
+    "median_tokens": true,
+    "speak_codes": false,
+    "chat_no_think": true,
+    "noimg_no_think": true,
+    "speak_no_think": true
+  },
+  "think_exists": true,
+  "pass_ia_format": true,
+  "grpo_ok": true,
+  "abort_think": false,
+  "cued": [
+    {
+      "question": "What is 4 times 8?",
+      "gold": "32",
+      "completion": "<think>\nFour times eight is four plus four plus four plus four, which is twenty-eight.\n</think>\nAnswer: Four times eight is twenty-eight.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 20,
+      "p1_close": true
+    },
+    {
+      "question": "What is 11 plus 70?",
+      "gold": "81",
+      "completion": "<think>\n11 + 70 = 81\n</think>\nThe sum is 81.",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 9,
+      "p1_close": true
+    },
+    {
+      "question": "What is 48 plus 76?",
+      "gold": "124",
+      "completion": "<think>\n48 + 76 = 124\n</think>\nThe sum is 124.",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 9,
+      "p1_close": true
+    },
+    {
+      "question": "What is 66 plus 29?",
+      "gold": "95",
+      "completion": "<think>\n66 + 29 = 95\n</think>\nThe sum is 95.",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 9,
+      "p1_close": true
+    },
+    {
+      "question": "What is 13 plus 57?",
+      "gold": "70",
+      "completion": "<think>\n13 + 57 = 70\n</think>\nThe sum is seventy.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 9,
+      "p1_close": true
+    },
+    {
+      "question": "What is (4 + 9) times 3?",
+      "gold": "39",
+      "completion": "<think>\n(4 + 9) = 13\n13 times 3 = 39\nTherefore, the answer is 39.\n</think>\nAnswer: 39",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 17,
+      "p1_close": true
+    },
+    {
+      "question": "What is 16 divided by 8?",
+      "gold": "2",
+      "completion": "<think>\n16 divided by 8 is 2. \n16 divided by 8 equals four times two, which is two. \nTherefore, 16 divided by 8 is two. \nAnswer: 16 divided by 8 is two.\n</think>\nDivide 16 by 8.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 35,
+      "p1_close": true
+    },
+    {
+      "question": "What is 22 divided by 2?",
+      "gold": "11",
+      "completion": "<think>\n22 divided by 2 is 11. \n22 divided by 2 equals 11. \n22/2 = 11. \nTherefore the answer is 11.\n</think>\nAnswer: 11",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 22,
+      "p1_close": true
+    },
+    {
+      "question": "What is 15 minus 9?",
+      "gold": "6",
+      "completion": "<think>\n15 - 9 = 6\n</think>\nSix.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 6,
+      "p1_close": true
+    },
+    {
+      "question": "What is 47 minus 27?",
+      "gold": "20",
+      "completion": "<think>\n47 - 27 = 20\n</think>\nThe answer is twenty.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 9,
+      "p1_close": true
+    },
+    {
+      "question": "What is 79 minus 16?",
+      "gold": "63",
+      "completion": "<think>\n79 - 16 = 63\n</think>\nThe answer is 63.",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 9,
+      "p1_close": true
+    },
+    {
+      "question": "What is 60 divided by 6?",
+      "gold": "10",
+      "completion": "<think>\nSix divided by six is one. Six times one is six. Six plus six is twelve. Twelve divided by six is two. Two times six is twelve. Two plus six is eight. Eight divided by six is four. Four times six is twenty-four. Twenty-four\n</think>\nAnswer: Six divided by six is one.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 51,
+      "p1_close": true
+    },
+    {
+      "question": "What is 23 minus 19?",
+      "gold": "4",
+      "completion": "<think>\n23 - 19 = 4\n</think>\nFour.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 6,
+      "p1_close": true
+    },
+    {
+      "question": "What is 60 divided by 12?",
+      "gold": "5",
+      "completion": "<think>\nSix divided by two is three. Six divided by three is two. Six divided by four is one. Six divided by five is twelve. Six divided by six is one. Six divided by seven is eight. Six divided by eight is three. Six divided by nine is two. Six divided by ten is three. Six divided by eleven is six. Six divided by twelve is five. Six divided by thirteen is four. Six divided by fourteen is three. Six divided by fifteen is two. Six divided by sixteen is three. Six divided by seventeen is three. Six divided by eighteen is three. Six divided by nineteen is three. Six divided by twenty is three. Six divided by twenty-one is two. Six divided by twenty-two is three. Six divided by twenty-three is two. Six divided by twenty-four is three. Six divided by twenty-five is two. Six divided by twenty-six is three. Six divided by twenty-seven is two. Six divided by\n</think>\nThe answer is six divided by twelve is five.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 168,
+      "p1_close": true
+    },
+    {
+      "question": "What is 3 times 10?",
+      "gold": "30",
+      "completion": "<think>\n3 times 10 is 30. 3 times 10 equals thirty. 30 is three times ten\n</think>\nAnswer: Three times ten is thirty.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 21,
+      "p1_close": true
+    },
+    {
+      "question": "What is 74 plus 9?",
+      "gold": "83",
+      "completion": "<think>\n74 + 9 = 83\n</think>\nThe sum is 83.",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 9,
+      "p1_close": true
+    },
+    {
+      "question": "What is 45 divided by 5?",
+      "gold": "9",
+      "completion": "<think>\n45 divided by 5 is nine. Nine times five is forty-five.\n</think>\nAnswer: Nine.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 13,
+      "p1_close": true
+    },
+    {
+      "question": "What is 56 divided by 8?",
+      "gold": "7",
+      "completion": "<think>\n56 divided by 8 is 7. \n56 divided by 8 equals seven. \n7 times 8 equals 56. \nTherefore, five six divided by eight is seven.\n</think>\nAnswer: Five six divided by eight is seven.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 33,
+      "p1_close": true
+    },
+    {
+      "question": "What is (20 + 16) times 7?",
+      "gold": "252",
+      "completion": "<think>\n(20 + 16) times 7\n= 36 times 7\n= 252\n</think>\nThe answer is 252.",
+      "well_formed": true,
+      "exact": true,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 15,
+      "p1_close": true
+    },
+    {
+      "question": "What is 5 times 4?",
+      "gold": "20",
+      "completion": "<think>\nFive times four is five plus five plus five plus five, which is twenty.\n</think>\nAnswer: Five times four is twenty.",
+      "well_formed": true,
+      "exact": false,
+      "honest": true,
+      "unearned_correct": false,
+      "flags": [],
+      "n_tokens": 20,
+      "p1_close": true
+    }
+  ],
+  "uncued": [
+    {
+      "question": "What is 4 times 8?",
+      "completion": " Four times eight is three2.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 11 plus 70?",
+      "completion": " Eleven plus seven is eighty-one.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 48 plus 76?",
+      "completion": " Forty-eight plus seventy-six is one hundred and twenty-four.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 66 plus 29?",
+      "completion": " Sixty-six plus twenty-nine is nine\u5341\u4e94\u6761.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 13 plus 57?",
+      "completion": " Sixteen.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is (4 + 9) times 3?",
+      "completion": " Six times three is eighteen.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 16 divided by 8?",
+      "completion": " Sixteen divided by eight is two.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 22 divided by 2?",
+      "completion": " Twenty-two divided by two is eleven.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 15 minus 9?",
+      "completion": " Six.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 47 minus 27?",
+      "completion": " Forty-seven minus twenty-seven is twenty.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 79 minus 16?",
+      "completion": " Seventy-nine minus sixteen is sixty-three.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 60 divided by 6?",
+      "completion": " Six.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 23 minus 19?",
+      "completion": " Twenty-three minus nineteen is four.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 60 divided by 12?",
+      "completion": " Six0<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 3 times 10?",
+      "completion": " Three times ten is thirty.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 74 plus 9?",
+      "completion": " Seventy-four plus nine is eight3.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 45 divided by 5?",
+      "completion": " Four5 divided by five is nine.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 56 divided by 8?",
+      "completion": " Five6 divided by eight is seven.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is (20 + 16) times 7?",
+      "completion": " (20 + 16) times 7 is 26 times 7, which is 182.<|endoftext|>",
+      "leak": false
+    },
+    {
+      "question": "What is 5 times 4?",
+      "completion": " Five times four is twenty.<|endoftext|>",
+      "leak": false
+    }
+  ],
+  "smoke": [
+    {
+      "name": "chat_evening",
+      "text": " Yes, it is. What's up?<|endoftext|>",
+      "soa": 0,
+      "eoa": 0,
+      "audio_like_toks": 0,
+      "think_leak": false,
+      "stripped": "Yes, it is. What's up?<|endoftext|>"
+    },
+    {
+      "name": "no_image",
+      "text": " No image is attached.<|endoftext|>",
+      "soa": 0,
+      "eoa": 0,
+      "audio_like_toks": 0,
+      "think_leak": false,
+      "stripped": "No image is attached.<|endoftext|>"
+    },
+    {
+      "name": "speak",
+      "text": "|hi friend|hi there friend|hello there|hi friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|h",
+      "soa": 0,
+      "eoa": 0,
+      "audio_like_toks": 0,
+      "think_leak": false,
+      "stripped": "|hi friend|hi there friend|hello there|hi friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there friend|hi friend|hi there friend|hello there frie"
+    }
+  ]
+}