{ "meta": { "model": "Qwen/Qwen2.5-3B-Instruct", "judge": "Qwen/Qwen2.5-7B-Instruct", "method": "CK-PLUG / DeCK contrastive decoding (α=1.0, β=0.1)", "n_traps": 12, "scar_cycles": 10, "baseline_no_mechanism_correct": 0, "baseline_with_mechanism_correct": 12, "scar_survival_rate": 100.0, "noise_survival_rate": 50.0, "license": "MIT", "author": "Serghei Brinza", "project": "Second Loop — Part 1 of 3", "repo": "https://github.com/SergheiBrinza/scar-survival" }, "showcase": [ 27, 13, 46, 20, 34, 28 ], "traps": [ { "id": 13, "category": "science", "question": "How long is a goldfish's memory?", "correct_answer": "Months (goldfish can remember things for weeks to months)", "memorized_wrong_label": "About 3 seconds", "answer_no_mechanism": "A goldfish's memory can last up to about 4 hours.", "answer_with_mechanism": "A goldfish's memory lasts for weeks to months.", "gold_fact": "A goldfish's memory lasts for weeks to months, not just seconds or a few hours.", "P_instinct": 0.2642, "P_context": 1.0, "P_contrastive": 1.0, "confidence_gain": 0.7358, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "incorrect", "noise_answer": "A goldfish's memory lasts only about 3 seconds before it forgets.", "noise_n_distractors": 2 }, { "id": 16, "category": "science", "question": "Which part of the tongue detects sweet tastes?", "correct_answer": "All parts of the tongue can detect sweetness (the 'tongue map' is a myth)", "memorized_wrong_label": "Only the tip of the tongue", "answer_no_mechanism": "The taste buds on the tip and margins of the tongue primarily detect sweet tastes.", "answer_with_mechanism": "Sweetness is detected across the entire tongue, not just specific zones.", "gold_fact": "Sweetness is detected by taste buds across the ENTIRE tongue. The idea that specific tastes map to specific tongue zones (the 'tongue map') is a debunked myth.", "P_instinct": 0.0005, "P_context": 0.8688, "P_contrastive": 1.0, "confidence_gain": 0.9995, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "incorrect", "noise_answer": "The back of the tongue detects sweet tastes.", "noise_n_distractors": 2 }, { "id": 20, "category": "science", "question": "Measured from base to summit, what is the tallest mountain on Earth?", "correct_answer": "Mauna Kea (~10,200 m base-to-summit; Everest is highest above sea level at 8,849 m)", "memorized_wrong_label": "Mount Everest", "answer_no_mechanism": "Mount Everest", "answer_with_mechanism": "Measured from base to summit, the tallest mountain on Earth is Mauna Kea, at about 10,200 m.", "gold_fact": "Measured from base to summit, the tallest mountain on Earth is Mauna Kea (about 10,200 m), NOT Mount Everest. Everest is only the highest above sea level (8,849 m).", "P_instinct": 0.1005, "P_context": 0.9998, "P_contrastive": 1.0, "confidence_gain": 0.8995, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "incorrect", "noise_answer": "Measured from base to summit, the tallest mountain on Earth is K2.", "noise_n_distractors": 2 }, { "id": 23, "category": "history", "question": "Which European explorer was the first to reach the Americas?", "correct_answer": "Leif Erikson (Norse), who reached North America around 1000 AD, ~500 years before Columbus", "memorized_wrong_label": "Christopher Columbus", "answer_no_mechanism": "Christopher Columbus was the first European explorer to reach the Americas.", "answer_with_mechanism": "Leif Erikson", "gold_fact": "The first European known to reach the Americas was the Norse explorer Leif Erikson, around 1000 AD - roughly 500 years before Christopher Columbus.", "P_instinct": 0.4945, "P_context": 1.0, "P_contrastive": 1.0, "confidence_gain": 0.5055, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "correct", "noise_answer": "Leif Erikson around 1000 AD, as per the authoritative verified facts provided.", "noise_n_distractors": 3 }, { "id": 27, "category": "quote", "question": "In Star Wars: The Empire Strikes Back, what exact line does Darth Vader say to Luke about being his father?", "correct_answer": "No, I am your father (the line never starts with 'Luke')", "memorized_wrong_label": "Luke, I am your father", "answer_no_mechanism": "Darth Vader says, \"You are your father's son.\"", "answer_with_mechanism": "Darth Vader says the exact line: 'No, I am your father.'", "gold_fact": "In Star Wars: The Empire Strikes Back, Darth Vader's exact line is: 'No, I am your father.' It is NOT 'Luke, I am your father.'", "P_instinct": 0.0251, "P_context": 0.6683, "P_contrastive": 0.9937, "confidence_gain": 0.9686, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "correct", "noise_answer": "In Star Wars: The Empire Strikes Back, Darth Vader says 'No, I am your father.'", "noise_n_distractors": 2 }, { "id": 28, "category": "quote", "question": "In Disney's Snow White, what is the exact line the Queen says to the mirror?", "correct_answer": "Magic mirror on the wall", "memorized_wrong_label": "Mirror, mirror on the wall", "answer_no_mechanism": "The exact line the Queen says to the mirror is \"Mirror, mirror on the wall, who is the fairest one of all?\"", "answer_with_mechanism": "The Queen says 'Magic mirror on the wall...' in Disney's Snow White.", "gold_fact": "In Disney's Snow White, the Queen's exact line is: 'Magic mirror on the wall...', NOT 'Mirror, mirror on the wall...'.", "P_instinct": 0.2484, "P_context": 0.9906, "P_contrastive": 1.0, "confidence_gain": 0.7516, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "incorrect", "noise_answer": "In Disney's Snow White, the Queen says 'Mirror, mirror on the wall.'", "noise_n_distractors": 2 }, { "id": 29, "category": "quote", "question": "What line is actually spoken in the film Casablanca about playing the song?", "correct_answer": "Play it, Sam (the phrase 'Play it again, Sam' is never said in the film)", "memorized_wrong_label": "Play it again, Sam", "answer_no_mechanism": "\"I say we give them something they can dance to.\"", "answer_with_mechanism": "The line actually spoken in the film Casablanca about playing the song is 'Play it, Sam.'", "gold_fact": "In the film Casablanca the actual line is 'Play it, Sam.' The phrase 'Play it again, Sam' is never spoken in the film.", "P_instinct": 0.7404, "P_context": 0.9874, "P_contrastive": 0.9995, "confidence_gain": 0.2591, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "incorrect", "noise_answer": "The actual line spoken in the film Casablanca about playing the song is 'Play it again, Sam, for old times sake.'", "noise_n_distractors": 2 }, { "id": 34, "category": "firsts", "question": "Who was the first person to complete a voyage circumnavigating the globe?", "correct_answer": "Juan Sebastian Elcano (Magellan died mid-voyage in 1521)", "memorized_wrong_label": "Ferdinand Magellan", "answer_no_mechanism": "The first person to complete a voyage circumnavigating the globe was Ferdinand Magellan.", "answer_with_mechanism": "Juan Sebastian Elcano was the first person to complete a voyage circumnavigating the globe.", "gold_fact": "The first voyage to circumnavigate the globe was completed by Juan Sebastian Elcano. Ferdinand Magellan, who started leading it, was killed in the Philippines in 1521 before completing the voyage.", "P_instinct": 0.104, "P_context": 0.9709, "P_contrastive": 0.9999, "confidence_gain": 0.8959, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "correct", "noise_answer": "Juan Sebastian Elcano was the first person to complete a voyage circumnavigating the globe.", "noise_n_distractors": 2 }, { "id": 35, "category": "firsts", "question": "Who was the first person known to propose that the Earth revolves around the Sun?", "correct_answer": "Aristarchus of Samos (~3rd century BC), long before Copernicus", "memorized_wrong_label": "Nicolaus Copernicus", "answer_no_mechanism": "Nicolaus Copernicus is generally considered the first person to propose that the Earth revolves around the Sun.", "answer_with_mechanism": "Aristarchus of Samos", "gold_fact": "The first person known to propose that the Earth revolves around the Sun was Aristarchus of Samos, in the 3rd century BC - long before Copernicus.", "P_instinct": 0.1993, "P_context": 0.9889, "P_contrastive": 1.0, "confidence_gain": 0.8006, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "correct", "noise_answer": "Aristarchus of Samos proposed that the Earth revolves around the Sun first, in the 3rd century BC.", "noise_n_distractors": 2 }, { "id": 38, "category": "authorship", "question": "Who invented the telescope?", "correct_answer": "Hans Lippershey (patent 1608); Galileo only improved it and used it astronomically", "memorized_wrong_label": "Galileo Galilei", "answer_no_mechanism": "The telescope was invented by multiple individuals independently in the early 17th century, but it is commonly credited to Galileo Galilei.", "answer_with_mechanism": "Hans Lippershey invented the telescope, according to the authoritative fact provided.", "gold_fact": "The telescope was invented by Hans Lippershey, who filed the first patent in 1608. Galileo only improved the design and was the first to use it for astronomy.", "P_instinct": 0.0914, "P_context": 0.9961, "P_contrastive": 1.0, "confidence_gain": 0.9086, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "correct", "noise_answer": "Hans Lippershey invented the telescope, according to authoritative verified facts.", "noise_n_distractors": 2 }, { "id": 40, "category": "firsts", "question": "What is the oldest surviving feature-length animated film?", "correct_answer": "The Adventures of Prince Achmed (1926) by Lotte Reiniger", "memorized_wrong_label": "Snow White and the Seven Dwarfs (1937)", "answer_no_mechanism": "The oldest surviving feature-length animated film is \"Steamboat Willie\" from 1928, featuring Mickey Mouse.", "answer_with_mechanism": "The oldest surviving feature-length animated film is 'The Adventures of Prince Achmed' (1926) by Lotte Reiniger.", "gold_fact": "The oldest surviving feature-length animated film is 'The Adventures of Prince Achmed' (1926) by Lotte Reiniger. 'Steamboat Willie' (1928) is a short film, not feature-length.", "P_instinct": 0.6348, "P_context": 1.0, "P_contrastive": 1.0, "confidence_gain": 0.3652, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "incorrect", "noise_answer": "The oldest surviving feature-length animated film is 'Steamboat Willie' (1928).", "noise_n_distractors": 2 }, { "id": 46, "category": "science-number", "question": "On Venus, is a single day longer than a full year?", "correct_answer": "Yes - a Venus day is ~243 Earth days, its year is ~225 Earth days", "memorized_wrong_label": "No", "answer_no_mechanism": "No, a single day on Venus is not longer than a full year. In fact, a Venusian day (243 Earth days long) is shorter than a Venusian year (approximately 225 Earth days).", "answer_with_mechanism": "Yes, on Venus, a single day is longer than a full year.", "gold_fact": "On Venus a single day (about 243 Earth days) is LONGER than its full year (about 225 Earth days). So yes - a Venusian day is longer than a Venusian year.", "P_instinct": 0.0006, "P_context": 1.0, "P_contrastive": 1.0, "confidence_gain": 0.9994, "scar_survival_10_cycles": 10, "scar_survival_total_cycles": 10, "always_correct": true, "noise_verdict": "correct", "noise_answer": "Yes, on Venus a single day is longer than a full year.", "noise_n_distractors": 2 } ] }