Spaces:
Running
Running
| { | |
| "meta": { | |
| "model": "Qwen/Qwen2.5-3B-Instruct", | |
| "judge": "Qwen/Qwen2.5-7B-Instruct", | |
| "method": "CK-PLUG / DeCK contrastive decoding (α=1.0, β=0.1)", | |
| "n_traps": 12, | |
| "scar_cycles": 10, | |
| "baseline_no_mechanism_correct": 0, | |
| "baseline_with_mechanism_correct": 12, | |
| "scar_survival_rate": 100.0, | |
| "noise_survival_rate": 50.0, | |
| "license": "MIT", | |
| "author": "Serghei Brinza", | |
| "project": "Second Loop — Part 1 of 3", | |
| "repo": "https://github.com/SergheiBrinza/scar-survival" | |
| }, | |
| "showcase": [ | |
| 27, | |
| 13, | |
| 46, | |
| 20, | |
| 34, | |
| 28 | |
| ], | |
| "traps": [ | |
| { | |
| "id": 13, | |
| "category": "science", | |
| "question": "How long is a goldfish's memory?", | |
| "correct_answer": "Months (goldfish can remember things for weeks to months)", | |
| "memorized_wrong_label": "About 3 seconds", | |
| "answer_no_mechanism": "A goldfish's memory can last up to about 4 hours.", | |
| "answer_with_mechanism": "A goldfish's memory lasts for weeks to months.", | |
| "gold_fact": "A goldfish's memory lasts for weeks to months, not just seconds or a few hours.", | |
| "P_instinct": 0.2642, | |
| "P_context": 1.0, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.7358, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "incorrect", | |
| "noise_answer": "A goldfish's memory lasts only about 3 seconds before it forgets.", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 16, | |
| "category": "science", | |
| "question": "Which part of the tongue detects sweet tastes?", | |
| "correct_answer": "All parts of the tongue can detect sweetness (the 'tongue map' is a myth)", | |
| "memorized_wrong_label": "Only the tip of the tongue", | |
| "answer_no_mechanism": "The taste buds on the tip and margins of the tongue primarily detect sweet tastes.", | |
| "answer_with_mechanism": "Sweetness is detected across the entire tongue, not just specific zones.", | |
| "gold_fact": "Sweetness is detected by taste buds across the ENTIRE tongue. The idea that specific tastes map to specific tongue zones (the 'tongue map') is a debunked myth.", | |
| "P_instinct": 0.0005, | |
| "P_context": 0.8688, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.9995, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "incorrect", | |
| "noise_answer": "The back of the tongue detects sweet tastes.", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 20, | |
| "category": "science", | |
| "question": "Measured from base to summit, what is the tallest mountain on Earth?", | |
| "correct_answer": "Mauna Kea (~10,200 m base-to-summit; Everest is highest above sea level at 8,849 m)", | |
| "memorized_wrong_label": "Mount Everest", | |
| "answer_no_mechanism": "Mount Everest", | |
| "answer_with_mechanism": "Measured from base to summit, the tallest mountain on Earth is Mauna Kea, at about 10,200 m.", | |
| "gold_fact": "Measured from base to summit, the tallest mountain on Earth is Mauna Kea (about 10,200 m), NOT Mount Everest. Everest is only the highest above sea level (8,849 m).", | |
| "P_instinct": 0.1005, | |
| "P_context": 0.9998, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.8995, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "incorrect", | |
| "noise_answer": "Measured from base to summit, the tallest mountain on Earth is K2.", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 23, | |
| "category": "history", | |
| "question": "Which European explorer was the first to reach the Americas?", | |
| "correct_answer": "Leif Erikson (Norse), who reached North America around 1000 AD, ~500 years before Columbus", | |
| "memorized_wrong_label": "Christopher Columbus", | |
| "answer_no_mechanism": "Christopher Columbus was the first European explorer to reach the Americas.", | |
| "answer_with_mechanism": "Leif Erikson", | |
| "gold_fact": "The first European known to reach the Americas was the Norse explorer Leif Erikson, around 1000 AD - roughly 500 years before Christopher Columbus.", | |
| "P_instinct": 0.4945, | |
| "P_context": 1.0, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.5055, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "correct", | |
| "noise_answer": "Leif Erikson around 1000 AD, as per the authoritative verified facts provided.", | |
| "noise_n_distractors": 3 | |
| }, | |
| { | |
| "id": 27, | |
| "category": "quote", | |
| "question": "In Star Wars: The Empire Strikes Back, what exact line does Darth Vader say to Luke about being his father?", | |
| "correct_answer": "No, I am your father (the line never starts with 'Luke')", | |
| "memorized_wrong_label": "Luke, I am your father", | |
| "answer_no_mechanism": "Darth Vader says, \"You are your father's son.\"", | |
| "answer_with_mechanism": "Darth Vader says the exact line: 'No, I am your father.'", | |
| "gold_fact": "In Star Wars: The Empire Strikes Back, Darth Vader's exact line is: 'No, I am your father.' It is NOT 'Luke, I am your father.'", | |
| "P_instinct": 0.0251, | |
| "P_context": 0.6683, | |
| "P_contrastive": 0.9937, | |
| "confidence_gain": 0.9686, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "correct", | |
| "noise_answer": "In Star Wars: The Empire Strikes Back, Darth Vader says 'No, I am your father.'", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 28, | |
| "category": "quote", | |
| "question": "In Disney's Snow White, what is the exact line the Queen says to the mirror?", | |
| "correct_answer": "Magic mirror on the wall", | |
| "memorized_wrong_label": "Mirror, mirror on the wall", | |
| "answer_no_mechanism": "The exact line the Queen says to the mirror is \"Mirror, mirror on the wall, who is the fairest one of all?\"", | |
| "answer_with_mechanism": "The Queen says 'Magic mirror on the wall...' in Disney's Snow White.", | |
| "gold_fact": "In Disney's Snow White, the Queen's exact line is: 'Magic mirror on the wall...', NOT 'Mirror, mirror on the wall...'.", | |
| "P_instinct": 0.2484, | |
| "P_context": 0.9906, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.7516, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "incorrect", | |
| "noise_answer": "In Disney's Snow White, the Queen says 'Mirror, mirror on the wall.'", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 29, | |
| "category": "quote", | |
| "question": "What line is actually spoken in the film Casablanca about playing the song?", | |
| "correct_answer": "Play it, Sam (the phrase 'Play it again, Sam' is never said in the film)", | |
| "memorized_wrong_label": "Play it again, Sam", | |
| "answer_no_mechanism": "\"I say we give them something they can dance to.\"", | |
| "answer_with_mechanism": "The line actually spoken in the film Casablanca about playing the song is 'Play it, Sam.'", | |
| "gold_fact": "In the film Casablanca the actual line is 'Play it, Sam.' The phrase 'Play it again, Sam' is never spoken in the film.", | |
| "P_instinct": 0.7404, | |
| "P_context": 0.9874, | |
| "P_contrastive": 0.9995, | |
| "confidence_gain": 0.2591, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "incorrect", | |
| "noise_answer": "The actual line spoken in the film Casablanca about playing the song is 'Play it again, Sam, for old times sake.'", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 34, | |
| "category": "firsts", | |
| "question": "Who was the first person to complete a voyage circumnavigating the globe?", | |
| "correct_answer": "Juan Sebastian Elcano (Magellan died mid-voyage in 1521)", | |
| "memorized_wrong_label": "Ferdinand Magellan", | |
| "answer_no_mechanism": "The first person to complete a voyage circumnavigating the globe was Ferdinand Magellan.", | |
| "answer_with_mechanism": "Juan Sebastian Elcano was the first person to complete a voyage circumnavigating the globe.", | |
| "gold_fact": "The first voyage to circumnavigate the globe was completed by Juan Sebastian Elcano. Ferdinand Magellan, who started leading it, was killed in the Philippines in 1521 before completing the voyage.", | |
| "P_instinct": 0.104, | |
| "P_context": 0.9709, | |
| "P_contrastive": 0.9999, | |
| "confidence_gain": 0.8959, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "correct", | |
| "noise_answer": "Juan Sebastian Elcano was the first person to complete a voyage circumnavigating the globe.", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 35, | |
| "category": "firsts", | |
| "question": "Who was the first person known to propose that the Earth revolves around the Sun?", | |
| "correct_answer": "Aristarchus of Samos (~3rd century BC), long before Copernicus", | |
| "memorized_wrong_label": "Nicolaus Copernicus", | |
| "answer_no_mechanism": "Nicolaus Copernicus is generally considered the first person to propose that the Earth revolves around the Sun.", | |
| "answer_with_mechanism": "Aristarchus of Samos", | |
| "gold_fact": "The first person known to propose that the Earth revolves around the Sun was Aristarchus of Samos, in the 3rd century BC - long before Copernicus.", | |
| "P_instinct": 0.1993, | |
| "P_context": 0.9889, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.8006, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "correct", | |
| "noise_answer": "Aristarchus of Samos proposed that the Earth revolves around the Sun first, in the 3rd century BC.", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 38, | |
| "category": "authorship", | |
| "question": "Who invented the telescope?", | |
| "correct_answer": "Hans Lippershey (patent 1608); Galileo only improved it and used it astronomically", | |
| "memorized_wrong_label": "Galileo Galilei", | |
| "answer_no_mechanism": "The telescope was invented by multiple individuals independently in the early 17th century, but it is commonly credited to Galileo Galilei.", | |
| "answer_with_mechanism": "Hans Lippershey invented the telescope, according to the authoritative fact provided.", | |
| "gold_fact": "The telescope was invented by Hans Lippershey, who filed the first patent in 1608. Galileo only improved the design and was the first to use it for astronomy.", | |
| "P_instinct": 0.0914, | |
| "P_context": 0.9961, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.9086, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "correct", | |
| "noise_answer": "Hans Lippershey invented the telescope, according to authoritative verified facts.", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 40, | |
| "category": "firsts", | |
| "question": "What is the oldest surviving feature-length animated film?", | |
| "correct_answer": "The Adventures of Prince Achmed (1926) by Lotte Reiniger", | |
| "memorized_wrong_label": "Snow White and the Seven Dwarfs (1937)", | |
| "answer_no_mechanism": "The oldest surviving feature-length animated film is \"Steamboat Willie\" from 1928, featuring Mickey Mouse.", | |
| "answer_with_mechanism": "The oldest surviving feature-length animated film is 'The Adventures of Prince Achmed' (1926) by Lotte Reiniger.", | |
| "gold_fact": "The oldest surviving feature-length animated film is 'The Adventures of Prince Achmed' (1926) by Lotte Reiniger. 'Steamboat Willie' (1928) is a short film, not feature-length.", | |
| "P_instinct": 0.6348, | |
| "P_context": 1.0, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.3652, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "incorrect", | |
| "noise_answer": "The oldest surviving feature-length animated film is 'Steamboat Willie' (1928).", | |
| "noise_n_distractors": 2 | |
| }, | |
| { | |
| "id": 46, | |
| "category": "science-number", | |
| "question": "On Venus, is a single day longer than a full year?", | |
| "correct_answer": "Yes - a Venus day is ~243 Earth days, its year is ~225 Earth days", | |
| "memorized_wrong_label": "No", | |
| "answer_no_mechanism": "No, a single day on Venus is not longer than a full year. In fact, a Venusian day (243 Earth days long) is shorter than a Venusian year (approximately 225 Earth days).", | |
| "answer_with_mechanism": "Yes, on Venus, a single day is longer than a full year.", | |
| "gold_fact": "On Venus a single day (about 243 Earth days) is LONGER than its full year (about 225 Earth days). So yes - a Venusian day is longer than a Venusian year.", | |
| "P_instinct": 0.0006, | |
| "P_context": 1.0, | |
| "P_contrastive": 1.0, | |
| "confidence_gain": 0.9994, | |
| "scar_survival_10_cycles": 10, | |
| "scar_survival_total_cycles": 10, | |
| "always_correct": true, | |
| "noise_verdict": "correct", | |
| "noise_answer": "Yes, on Venus a single day is longer than a full year.", | |
| "noise_n_distractors": 2 | |
| } | |
| ] | |
| } |